From 17b2f35ae71daeccadf8f1eb6b64413c6b0ed643 Mon Sep 17 00:00:00 2001 From: Mick Date: Fri, 9 Oct 2026 16:30:55 +0800 Subject: [PATCH] [diffusion] refactor: remove unused helpers and write-only state (#43064) Co-authored-by: Mick Qian --- .../ComfyUI_SGLDiffusion/core/server_api.py | 35 ---- .../sglang/multimodal_gen/apps/webui/main.py | 8 - .../configs/models/dits/krea2.py | 5 - .../configs/models/dits/ltx_2.py | 6 - .../configs/models/encoders/ideogram.py | 3 - .../configs/models/vaes/base.py | 13 ++ .../configs/models/vaes/ltx_video.py | 29 ++++ .../configs/pipeline_configs/base.py | 42 ++--- .../pipeline_configs/diffusers_generic.py | 6 - .../configs/pipeline_configs/flux.py | 6 +- .../configs/pipeline_configs/joy_image.py | 29 +--- .../configs/pipeline_configs/krea2.py | 14 +- .../configs/pipeline_configs/longcat_image.py | 16 +- .../configs/pipeline_configs/ltx_2.py | 59 +------ .../configs/pipeline_configs/minimax_h3.py | 7 - .../configs/pipeline_configs/mova.py | 13 -- .../configs/pipeline_configs/qwen_image.py | 57 ++----- .../configs/pipeline_configs/sana_wm.py | 42 +---- .../multimodal_gen/configs/sample/action.py | 11 +- .../configs/sample/minimax_h3.py | 17 +- .../configs/sample/sampling_params.py | 4 - python/sglang/multimodal_gen/configs/utils.py | 25 ++- python/sglang/multimodal_gen/envs.py | 14 +- .../model_padders/ideogram.py | 10 +- .../model_padders/zimage.py | 10 +- .../breakable_cuda_graph/prompt_padding.py | 8 + .../runtime/disaggregation/disagg_args.py | 28 --- .../runtime/disaggregation/dispatch_policy.py | 78 +-------- .../runtime/disaggregation/metrics.py | 16 +- .../runtime/disaggregation/orchestrator.py | 87 +++++----- .../runtime/disaggregation/scheduler_mixin.py | 2 - .../runtime/distributed/communication_op.py | 5 - .../base_device_communicator.py | 1 - .../runtime/distributed/group_coordinator.py | 6 - .../runtime/distributed/parallel_state.py | 72 -------- .../runtime/entrypoints/cli/generate.py | 10 -- .../runtime/entrypoints/openai/protocol.py | 4 - .../adapters/sana_wm_realtime_adapter.py | 1 - .../runtime/layers/activation.py | 9 - .../layers/attention/backends/ascend_fa.py | 6 - .../attention/backends/attention_backend.py | 10 -- .../attention/backends/block_sparse_attn.py | 6 - .../backends/cube_sparse_attn/backend.py | 6 - .../layers/attention/backends/flash_attn.py | 7 - .../backends/hybrid_window_attn_h3.py | 6 - .../attention/backends/rain_fusion_attn.py | 6 - .../attention/backends/sliding_tile_attn.py | 6 - .../attention/backends/sparse_linear_attn.py | 91 +++------- .../backends/sparse_video_gen_2_attn.py | 14 -- .../backends/subblock_sparse_attn.py | 8 - .../attention/backends/video_sparse_attn.py | 7 - .../backends/video_sparse_attn_h3.py | 3 - .../layers/attention/backends/vmoba.py | 6 - .../runtime/layers/custom_op.py | 8 - .../multimodal_gen/runtime/layers/linear.py | 68 +++----- .../runtime/layers/lora/linear.py | 70 -------- .../runtime/layers/parallel_conv.py | 4 +- .../convrot_int8_comfy_kitchen.py | 28 +-- .../layers/quantization/convrot_int8_jit.py | 24 ++- .../layers/quantization/modelopt_quant.py | 47 +---- .../runtime/layers/quantization/mxfp4_npu.py | 3 - .../layers/rotary_embedding/factory.py | 1 - .../runtime/layers/rotary_embedding/mrope.py | 20 +++ .../multimodal_gen/runtime/layers/usp.py | 11 -- .../multimodal_gen/runtime/layers/utils.py | 11 +- .../layers/vocab_parallel_embedding.py | 91 +--------- .../component_loaders/text_encoder_loader.py | 5 +- .../loader/component_loaders/vae_loader.py | 2 - .../runtime/loader/weight_utils.py | 90 +++++----- .../runtime/managers/forward_context.py | 70 +------- .../memory_managers/auto_residency.py | 160 +----------------- .../component_loading_order.py | 10 +- .../runtime/models/dits/causal_wanvideo.py | 1 - .../runtime/models/dits/cosmos3video.py | 60 +++---- .../runtime/models/dits/flux.py | 1 - .../runtime/models/dits/flux_2.py | 1 - .../runtime/models/dits/hunyuanvideo.py | 1 - .../runtime/models/dits/lingbot_world.py | 11 +- .../runtime/models/dits/ltx_2.py | 8 - .../runtime/models/dits/mova_audio_dit.py | 36 +--- .../runtime/models/dits/mova_video_dit.py | 64 ++++--- .../runtime/models/dits/qwen_image.py | 1 - .../runtime/models/dits/sana_wm.py | 38 ++--- .../runtime/models/dits/wanvideo.py | 15 +- .../runtime/models/dits/zimage.py | 4 - .../runtime/models/encoders/base.py | 14 +- .../runtime/models/encoders/gemma2.py | 11 +- .../runtime/models/encoders/gemma_3.py | 41 ++++- .../runtime/models/encoders/llama.py | 53 ++---- .../runtime/models/encoders/qwen2_5vl.py | 11 +- .../runtime/models/encoders/qwen3.py | 56 ++---- .../runtime/models/encoders/t5.py | 4 - .../models/schedulers/flow_match_pair.py | 98 ++--------- .../runtime/models/vaes/alias_free.py | 18 +- .../models/vaes/autoencoder_kl_qwenimage.py | 12 -- .../runtime/models/vaes/common.py | 92 ++++------ .../runtime/models/vaes/cosmos3_avae.py | 6 - .../multimodal_gen/runtime/models/vaes/dac.py | 1 - .../runtime/models/vaes/hunyuan3d_vae.py | 6 - .../models/vaes/ltx_2_3_condition_encoder.py | 1 - .../runtime/models/vaes/ltx_2_audio.py | 3 - .../runtime/models/vaes/ltx_2_vae.py | 13 +- .../vaes/minimax_h3_audio_vae/audio_vae.py | 1 - .../vaes/minimax_h3_audio_vae/bigvgan.py | 1 - .../models/vaes/minimax_h3_video_vae/klvae.py | 1 - .../vaes/minimax_h3_video_vae/vae_cnn.py | 2 - .../runtime/models/vaes/wanvae.py | 2 +- .../runtime/models/vlas/pi05_core.py | 16 -- .../runtime/models/vocoder/ltx_2_vocoder.py | 1 - .../runtime/pipelines/diffusers_pipeline.py | 3 - .../runtime/pipelines/ltx_2_pipeline.py | 6 - .../runtime/pipelines_core/stages/base.py | 18 -- .../pipelines_core/stages/denoising.py | 4 - .../stages/latent_preparation.py | 43 ++--- .../stages/model_specific_stages/glm_image.py | 8 - .../model_specific_stages/helios_denoising.py | 9 +- .../model_specific_stages/hunyuan3d/shape.py | 3 - .../kandinsky6/decoding.py | 3 - .../kandinsky6_sr/tiling.py | 1 - .../lingbot_video_moe/text_encoding.py | 2 - .../minimax_h3/request_validation.py | 16 +- .../minimax_h3/stages/denoising.py | 1 - .../minimax_h3/stages/latent_preparation.py | 23 +-- .../stages/model_specific_stages/mova.py | 1 - .../qwen_image_layered.py | 1 - .../stages/progressive_resolution/wan.py | 15 -- .../stages/realtime/text_encoding.py | 22 +-- .../multimodal_gen/runtime/platforms/cuda.py | 9 - .../runtime/platforms/interface.py | 5 - .../multimodal_gen/runtime/platforms/mps.py | 10 -- .../multimodal_gen/runtime/platforms/musa.py | 9 - .../multimodal_gen/runtime/platforms/npu.py | 9 - .../multimodal_gen/runtime/platforms/rocm.py | 4 - .../multimodal_gen/runtime/platforms/xpu.py | 11 -- .../runtime/post_training/sp_utils.py | 9 - .../postprocess/realesrgan_upscaler.py | 16 +- .../runtime/postprocess/rife_interpolator.py | 3 - .../multimodal_gen/runtime/realtime/video.py | 1 - .../runtime/server_args/disagg.py | 7 +- .../runtime/server_args/server_args.py | 7 - .../multimodal_gen/runtime/utils/common.py | 14 -- .../runtime/utils/hf_diffusers_utils.py | 60 ------- .../runtime/utils/logging_utils.py | 124 +------------- .../runtime/utils/mesh3d_utils.py | 2 - .../runtime/utils/quantization_utils.py | 39 +---- .../multimodal_gen/runtime/utils/vision.py | 4 +- .../single_test_file/test_disagg_server.py | 2 +- .../sglang/multimodal_gen/test/test_utils.py | 98 +---------- .../test/unit/test_cache_dit_per_request.py | 1 - .../test/unit/test_server_args.py | 12 -- 150 files changed, 597 insertions(+), 2456 deletions(-) delete mode 100644 python/sglang/multimodal_gen/runtime/disaggregation/disagg_args.py diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/server_api.py b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/server_api.py index 944d93ca82..7e90f51fb2 100644 --- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/server_api.py +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/server_api.py @@ -3,14 +3,11 @@ SGLang Diffusion Server API client. Provides a low-level interface for interacting with SGLang Diffusion HTTP server. """ -import base64 -import io import os import time from typing import Any, Dict, Optional import requests -from PIL import Image class SGLDiffusionServerAPI: @@ -417,38 +414,6 @@ class SGLDiffusionServerAPI: } return content_types.get(ext, "image/png") - def decode_image_from_response( - self, response_data: Dict[str, Any], index: int = 0 - ) -> Image.Image: - """ - Decode base64 image from API response. - - Args: - response_data: API response dictionary - index: Index of the image in the response (default: 0) - - Returns: - PIL Image object - """ - if "data" not in response_data or not response_data["data"]: - raise ValueError("No image data in response") - - if index >= len(response_data["data"]): - raise IndexError(f"Image index {index} out of range") - - image_data = response_data["data"][index] - if "b64_json" not in image_data or not image_data["b64_json"]: - raise ValueError("No base64 image data found") - - image_bytes = base64.b64decode(image_data["b64_json"]) - image = Image.open(io.BytesIO(image_bytes)) - - # Convert to RGB if needed - if image.mode != "RGB": - image = image.convert("RGB") - - return image - def set_lora( self, lora_nickname: str, diff --git a/python/sglang/multimodal_gen/apps/webui/main.py b/python/sglang/multimodal_gen/apps/webui/main.py index 1391130377..4018c05fba 100644 --- a/python/sglang/multimodal_gen/apps/webui/main.py +++ b/python/sglang/multimodal_gen/apps/webui/main.py @@ -1,4 +1,3 @@ -import argparse import os from sglang.multimodal_gen.apps.webui.minimax_h3 import ( @@ -21,13 +20,6 @@ from sglang.srt.environ import envs logger = init_logger(__name__) -def add_webui_args(parser: argparse.ArgumentParser): - """Add the arguments for the generate command.""" - parser = ServerArgs.add_cli_args(parser) - parser = SamplingParams.add_cli_args(parser) - return parser - - def run_sgl_diffusion_webui(server_args: ServerArgs): if is_minimax_h3(server_args): return run_minimax_h3_webui(server_args) diff --git a/python/sglang/multimodal_gen/configs/models/dits/krea2.py b/python/sglang/multimodal_gen/configs/models/dits/krea2.py index b330b53642..d103b695d7 100644 --- a/python/sglang/multimodal_gen/configs/models/dits/krea2.py +++ b/python/sglang/multimodal_gen/configs/models/dits/krea2.py @@ -65,11 +65,6 @@ class Krea2ArchConfig(DiTArchConfig): def head_dim(self) -> int: return self.features // self.heads - @property - def in_features_packed(self) -> int: - """Patch-embed input width: channels * patch**2.""" - return self.channels * self.patch**2 - @dataclass class Krea2DitConfig(DiTConfig): diff --git a/python/sglang/multimodal_gen/configs/models/dits/ltx_2.py b/python/sglang/multimodal_gen/configs/models/dits/ltx_2.py index 42ed057cda..81b5355c25 100644 --- a/python/sglang/multimodal_gen/configs/models/dits/ltx_2.py +++ b/python/sglang/multimodal_gen/configs/models/dits/ltx_2.py @@ -17,12 +17,6 @@ class LTXModelType(Enum): VideoOnly = "ltx video only model" AudioOnly = "ltx audio only model" - def is_video_enabled(self) -> bool: - return self in (LTXModelType.AudioVideo, LTXModelType.VideoOnly) - - def is_audio_enabled(self) -> bool: - return self in (LTXModelType.AudioVideo, LTXModelType.AudioOnly) - class LTX2RopeType(str, Enum): """ diff --git a/python/sglang/multimodal_gen/configs/models/encoders/ideogram.py b/python/sglang/multimodal_gen/configs/models/encoders/ideogram.py index fab9713728..5313a47f67 100644 --- a/python/sglang/multimodal_gen/configs/models/encoders/ideogram.py +++ b/python/sglang/multimodal_gen/configs/models/encoders/ideogram.py @@ -31,7 +31,4 @@ class Ideogram4TextEncoderConfig(Qwen3VLConfig): self.arch_config.ideogram_fp8_weight_only = not use_bitsandbytes self.arch_config.requires_gpu_resident_text_encoder = use_bitsandbytes - def finalize_model_arch(self): - self.post_diffusers_config_update() - arch_config: Qwen3VLArchConfig = field(default_factory=Qwen3VLArchConfig) diff --git a/python/sglang/multimodal_gen/configs/models/vaes/base.py b/python/sglang/multimodal_gen/configs/models/vaes/base.py index 934a02219b..6afc5d48fb 100644 --- a/python/sglang/multimodal_gen/configs/models/vaes/base.py +++ b/python/sglang/multimodal_gen/configs/models/vaes/base.py @@ -16,6 +16,19 @@ AUTO_PARALLEL_DECODE_MODE = "auto" SPATIAL_SHARD_PARALLEL_DECODE_MODES = ("spatial_shard", "spatial") +def get_channelwise_decode_scale_and_shift(arch_config, device, dtype): + """Return reciprocal std and mean broadcast over BCTHW latents.""" + scaling_factor = 1.0 / torch.tensor(arch_config.latents_std, device=device).view( + 1, arch_config.z_dim, 1, 1, 1 + ).to(device, dtype) + shift_factor = ( + torch.tensor(arch_config.latents_mean) + .view(1, arch_config.z_dim, 1, 1, 1) + .to(device, dtype) + ) + return scaling_factor, shift_factor + + @lru_cache(maxsize=8) def is_spatial_shard_parallel_decode_mode(mode: str) -> bool: return mode in SPATIAL_SHARD_PARALLEL_DECODE_MODES diff --git a/python/sglang/multimodal_gen/configs/models/vaes/ltx_video.py b/python/sglang/multimodal_gen/configs/models/vaes/ltx_video.py index 8e36617289..aa91f063bd 100644 --- a/python/sglang/multimodal_gen/configs/models/vaes/ltx_video.py +++ b/python/sglang/multimodal_gen/configs/models/vaes/ltx_video.py @@ -2,9 +2,38 @@ from dataclasses import dataclass, field from typing import Any, List +import torch + from sglang.multimodal_gen.configs.models.vaes.base import VAEArchConfig, VAEConfig +def get_ltx_video_decode_scale_and_shift(device, dtype, vae, arch_config): + """Invert LTX video normalization: (z - mean) * scaling_factor / std.""" + latents_mean = getattr(vae, "latents_mean", None) + latents_std = getattr(vae, "latents_std", None) + scaling_factor = ( + getattr(getattr(vae, "config", None), "scaling_factor", None) + or getattr(vae, "scaling_factor", None) + or getattr(arch_config, "scaling_factor", None) + or 1.0 + ) + if isinstance(scaling_factor, (int, float)) and float(scaling_factor) == 0.0: + scaling_factor = 1.0 + + if isinstance(latents_mean, torch.Tensor) and isinstance(latents_std, torch.Tensor): + latents_mean = latents_mean.to(device=device, dtype=dtype).view(1, -1, 1, 1, 1) + latents_std = latents_std.to(device=device, dtype=dtype).view(1, -1, 1, 1, 1) + sf = torch.tensor(float(scaling_factor), device=device, dtype=dtype).view( + 1, 1, 1, 1, 1 + ) + return sf / latents_std, latents_mean + + sf = torch.tensor(float(scaling_factor), device=device, dtype=dtype).view( + 1, 1, 1, 1, 1 + ) + return sf, None + + @dataclass class LTXVideoVAEArchConfig(VAEArchConfig): # Architecture params diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/base.py b/python/sglang/multimodal_gen/configs/pipeline_configs/base.py index f126207ae6..7ad38d1ad9 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/base.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/base.py @@ -5,9 +5,8 @@ import json import math import os from collections.abc import Callable -from dataclasses import asdict, dataclass, field, fields +from dataclasses import dataclass, field, fields from enum import Enum -from operator import attrgetter from typing import Any, ClassVar import numpy as np @@ -56,6 +55,16 @@ class STA_Mode(str, Enum): NONE = None +def pack_latents_2x2(latents, batch_size, num_channels_latents, height, width): + latents = latents.view( + batch_size, num_channels_latents, height // 2, 2, width // 2, 2 + ) + latents = latents.permute(0, 2, 4, 1, 3, 5) + return latents.reshape( + batch_size, (height // 2) * (width // 2), num_channels_latents * 4 + ) + + def postprocess_text(output: BaseEncoderOutput, _text_inputs) -> torch.tensor: raise NotImplementedError @@ -1198,35 +1207,6 @@ class PipelineConfig: f"Length of text postprocess functions ({len(self.postprocess_text_funcs)}) must be equal to length of text preprocessing functions ({len(self.preprocess_text_funcs)})" ) - def dump_to_json(self, file_path: str): - output_dict = {f.name: attrgetter(f.name)(self) for f in fields(self)} - del_keys = [] - for key, value in output_dict.items(): - if isinstance(value, ModelConfig): - model_dict = asdict(value) - # Model Arch Config should be hidden away from the users - model_dict.pop("arch_config") - output_dict[key] = model_dict - elif isinstance(value, tuple) and all( - isinstance(v, ModelConfig) for v in value - ): - model_dicts = [] - for v in value: - model_dict = asdict(v) - # Model Arch Config should be hidden away from the users - model_dict.pop("arch_config") - model_dicts.append(model_dict) - output_dict[key] = model_dicts - elif isinstance(value, tuple) and all(callable(f) for f in value): - # Skip dumping functions - del_keys.append(key) - - for key in del_keys: - output_dict.pop(key, None) - - with open(file_path, "w") as f: - json.dump(output_dict, f, indent=2) - def load_from_json(self, file_path: str): with open(file_path) as f: input_pipeline_dict = json.load(f) diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/diffusers_generic.py b/python/sglang/multimodal_gen/configs/pipeline_configs/diffusers_generic.py index b3a3e95721..734f864272 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/diffusers_generic.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/diffusers_generic.py @@ -67,12 +67,6 @@ class DiffusersGenericPipelineConfig(PipelineConfig): """ pass - def adjust_size(self, width, height, image): - """ - Pass through - diffusers handles size adjustments. - """ - return width, height - # Static subclasses for each non-default task type. # These exist so that _get_diffusers_model_info() can swap the task_type without diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/flux.py b/python/sglang/multimodal_gen/configs/pipeline_configs/flux.py index 0b344904b4..8d21318f8a 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/flux.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/flux.py @@ -20,12 +20,12 @@ from sglang.multimodal_gen.configs.models.vaes.flux import Flux2VAEConfig, FluxV from sglang.multimodal_gen.configs.pipeline_configs.base import ( ImagePipelineConfig, ModelTaskType, + pack_latents_2x2, shard_rotary_emb_for_sp, ) from sglang.multimodal_gen.configs.pipeline_configs.hunyuan import ( clip_postprocess_text, ) -from sglang.multimodal_gen.configs.pipeline_configs.qwen_image import _pack_latents from sglang.multimodal_gen.runtime.distributed import get_local_torch_device @@ -141,7 +141,9 @@ class FluxPipelineConfig(ImagePipelineConfig): width = 2 * (batch.width // (self.vae_config.arch_config.vae_scale_factor * 2)) num_channels_latents = self.dit_config.arch_config.in_channels // 4 # pack latents - return _pack_latents(latents, batch_size, num_channels_latents, height, width) + return pack_latents_2x2( + latents, batch_size, num_channels_latents, height, width + ) def get_pos_prompt_embeds(self, batch): return batch.prompt_embeds[1] diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/joy_image.py b/python/sglang/multimodal_gen/configs/pipeline_configs/joy_image.py index 2e68c4851a..4773d11a30 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/joy_image.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/joy_image.py @@ -11,6 +11,9 @@ from sglang.multimodal_gen.configs.models import DiTConfig, EncoderConfig, VAECo from sglang.multimodal_gen.configs.models.dits.joy_image import JoyImageDiTConfig from sglang.multimodal_gen.configs.models.encoders.qwen3vl import Qwen3VLConfig from sglang.multimodal_gen.configs.models.vaes import WanVAEConfig +from sglang.multimodal_gen.configs.models.vaes.base import ( + get_channelwise_decode_scale_and_shift, +) from sglang.multimodal_gen.configs.pipeline_configs.base import ( ImagePipelineConfig, ModelTaskType, @@ -249,32 +252,10 @@ class JoyImageEditPipelineConfig(ImagePipelineConfig): def get_decode_scale_and_shift( self, device, dtype, vae ) -> Tuple[torch.Tensor, torch.Tensor]: - """Get VAE denormalization scale and shift. - - Args: - device: Target device - dtype: Target dtype - vae: VAE model - - Returns: - Tuple of (scaling_factor, shift_factor) - """ - vae_arch_config = self.vae_config.arch_config - - # Create scale factor: 1.0 / std - scaling_factor = 1.0 / torch.tensor( - vae_arch_config.latents_std, device=device - ).view(1, vae_arch_config.z_dim, 1, 1, 1).to(device, dtype) - - # Create shift factor: mean - shift_factor = ( - torch.tensor(vae_arch_config.latents_mean) - .view(1, vae_arch_config.z_dim, 1, 1, 1) - .to(device, dtype) + return get_channelwise_decode_scale_and_shift( + self.vae_config.arch_config, device, dtype ) - return scaling_factor, shift_factor - def prepare_calculated_size(self, img: Image.Image) -> Tuple[int, int]: img_h, img_w = img.size[1], img.size[0] # PIL (w,h) bucket = self.find_best_bucket((1, 1, img_h, img_w)) diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/krea2.py b/python/sglang/multimodal_gen/configs/pipeline_configs/krea2.py index 104ce82455..e726974f29 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/krea2.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/krea2.py @@ -6,6 +6,9 @@ from diffusers.image_processor import VaeImageProcessor from sglang.multimodal_gen.configs.models import DiTConfig, VAEConfig from sglang.multimodal_gen.configs.models.dits.krea2 import Krea2DitConfig +from sglang.multimodal_gen.configs.models.vaes.base import ( + get_channelwise_decode_scale_and_shift, +) from sglang.multimodal_gen.configs.models.vaes.qwenimage import QwenImageVAEConfig from sglang.multimodal_gen.configs.pipeline_configs.base import ( ImagePipelineConfig, @@ -146,16 +149,9 @@ class Krea2PipelineConfig(ImagePipelineConfig): # --- VAE decode (same as Qwen-Image: latents * std + mean) --- def get_decode_scale_and_shift(self, device, dtype, vae): - vae_arch = self.vae_config.arch_config - scaling_factor = 1.0 / torch.tensor(vae_arch.latents_std, device=device).view( - 1, vae_arch.z_dim, 1, 1, 1 - ).to(device, dtype) - shift_factor = ( - torch.tensor(vae_arch.latents_mean) - .view(1, vae_arch.z_dim, 1, 1, 1) - .to(device, dtype) + return get_channelwise_decode_scale_and_shift( + self.vae_config.arch_config, device, dtype ) - return scaling_factor, shift_factor def post_denoising_loop(self, latents, batch): latents, batch_size, channels, height, width = self._unpad_and_unpack_latents( diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/longcat_image.py b/python/sglang/multimodal_gen/configs/pipeline_configs/longcat_image.py index 822e2a6374..f0d480c487 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/longcat_image.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/longcat_image.py @@ -17,6 +17,7 @@ from sglang.multimodal_gen.configs.pipeline_configs.base import ( ImagePipelineConfig, ModelTaskType, TextConditioningOutput, + pack_latents_2x2, ) from sglang.multimodal_gen.configs.pipeline_configs.model_deployment_config import ( ModelDeploymentConfig, @@ -171,17 +172,6 @@ def _unpack_latents(latents, height, width, vae_scale_factor): return latents -def _pack_latents(latents, batch_size, num_channels_latents, height, width): - latents = latents.view( - batch_size, num_channels_latents, height // 2, 2, width // 2, 2 - ) - latents = latents.permute(0, 2, 4, 1, 3, 5) - latents = latents.reshape( - batch_size, (height // 2) * (width // 2), num_channels_latents * 4 - ) - return latents - - def _calculate_shift( image_seq_len, base_seq_len: int = 256, @@ -293,7 +283,7 @@ class LongCatImagePipelineConfig(ImagePipelineConfig): def maybe_pack_latents(self, latents, batch_size, batch): num_channels_latents = self.dit_config.arch_config.num_channels_latents _, _, h, w = latents.shape - return _pack_latents(latents, batch_size, num_channels_latents, h, w) + return pack_latents_2x2(latents, batch_size, num_channels_latents, h, w) def maybe_prepare_latent_ids(self, latents): # latents shape after packing: [B, (h//2)*(w//2), C*4] @@ -517,7 +507,7 @@ class LongCatImageEditPipelineConfig(LongCatImagePipelineConfig): batch_size // latent_condition.shape[0], 1, 1, 1 ) _, num_channels_latents, height, width = latent_condition.shape - return _pack_latents( + return pack_latents_2x2( latent_condition, batch_size, num_channels_latents, height, width ) diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/ltx_2.py b/python/sglang/multimodal_gen/configs/pipeline_configs/ltx_2.py index b127d8c8e4..872dc86909 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/ltx_2.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/ltx_2.py @@ -11,7 +11,10 @@ from sglang.multimodal_gen.configs.models.encoders import ( ) from sglang.multimodal_gen.configs.models.encoders.gemma_3 import Gemma3Config from sglang.multimodal_gen.configs.models.vaes.ltx_audio import LTXAudioVAEConfig -from sglang.multimodal_gen.configs.models.vaes.ltx_video import LTXVideoVAEConfig +from sglang.multimodal_gen.configs.models.vaes.ltx_video import ( + LTXVideoVAEConfig, + get_ltx_video_decode_scale_and_shift, +) from sglang.multimodal_gen.configs.pipeline_configs.base import ( ModelTaskType, PipelineConfig, @@ -564,36 +567,9 @@ class LTX2PipelineConfig(PipelineConfig): ) def get_decode_scale_and_shift(self, device, dtype, vae): - latents_mean = getattr(vae, "latents_mean", None) - latents_std = getattr(vae, "latents_std", None) - - scaling_factor = ( - getattr(getattr(vae, "config", None), "scaling_factor", None) - or getattr(vae, "scaling_factor", None) - or getattr(self.vae_config.arch_config, "scaling_factor", None) - or 1.0 + return get_ltx_video_decode_scale_and_shift( + device, dtype, vae, self.vae_config.arch_config ) - if isinstance(scaling_factor, (int, float)) and float(scaling_factor) == 0.0: - scaling_factor = 1.0 - - if isinstance(latents_mean, torch.Tensor) and isinstance( - latents_std, torch.Tensor - ): - latents_mean = latents_mean.to(device=device, dtype=dtype).view( - 1, -1, 1, 1, 1 - ) - latents_std = latents_std.to(device=device, dtype=dtype).view( - 1, -1, 1, 1, 1 - ) - sf = torch.tensor(float(scaling_factor), device=device, dtype=dtype).view( - 1, 1, 1, 1, 1 - ) - return sf / latents_std, latents_mean - - sf = torch.tensor(float(scaling_factor), device=device, dtype=dtype).view( - 1, 1, 1, 1, 1 - ) - return sf, None @staticmethod def _unpack_latents( @@ -626,29 +602,6 @@ class LTX2PipelineConfig(PipelineConfig): ) return latents - @staticmethod - def _denormalize_latents( - latents: torch.Tensor, - latents_mean: torch.Tensor, - latents_std: torch.Tensor, - scaling_factor: float = 1.0, - ) -> torch.Tensor: - # Denormalize latents across the channel dimension [B, C, F, H, W] - latents_mean = latents_mean.view(1, -1, 1, 1, 1).to( - latents.device, latents.dtype - ) - latents_std = latents_std.view(1, -1, 1, 1, 1).to(latents.device, latents.dtype) - latents = latents * latents_std / scaling_factor + latents_mean - return latents - - @staticmethod - def _denormalize_audio_latents( - latents: torch.Tensor, latents_mean: torch.Tensor, latents_std: torch.Tensor - ): - latents_mean = latents_mean.to(latents.device, latents.dtype) - latents_std = latents_std.to(latents.device, latents.dtype) - return (latents * latents_std) + latents_mean - @staticmethod def _unpack_audio_latents( latents: torch.Tensor, diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/minimax_h3.py b/python/sglang/multimodal_gen/configs/pipeline_configs/minimax_h3.py index c99edc832c..e4da4079a3 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/minimax_h3.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/minimax_h3.py @@ -70,16 +70,9 @@ class MiniMaxH3PipelineConfig(PipelineConfig): output_audio_channels: int | None = 2 output_av_drift_tolerance_s: float | None = 0.25 - def accepts_audio_input(self) -> bool: - return True - def supports_disaggregation(self) -> bool: return False - @property - def requires_audio_output(self) -> bool: - return True - def get_model_deployment_config(self) -> ModelDeploymentConfig: return ModelDeploymentConfig( speed_mode_enable_torch_compile_by_default=False, diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/mova.py b/python/sglang/multimodal_gen/configs/pipeline_configs/mova.py index 7e09301bc7..9371a778eb 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/mova.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/mova.py @@ -154,19 +154,6 @@ class MOVAPipelineConfig(PipelineConfig): latent_T = (num_samples + audio_vae.hop_length - 1) // audio_vae.hop_length return (batch_size, audio_vae.latent_dim, latent_T) - def normalize_video_latents(self, latents: torch.Tensor, video_vae) -> torch.Tensor: - latents_mean = getattr(video_vae.config, "latents_mean", None) - latents_std = getattr(video_vae.config, "latents_std", None) - if latents_mean is None or latents_std is None: - return latents - mean = torch.tensor( - latents_mean, device=latents.device, dtype=latents.dtype - ).view(1, video_vae.config.z_dim, 1, 1, 1) - inv_std = ( - 1.0 / torch.tensor(latents_std, device=latents.device, dtype=latents.dtype) - ).view(1, video_vae.config.z_dim, 1, 1, 1) - return (latents - mean) * inv_std - def denormalize_video_latents( self, latents: torch.Tensor, video_vae ) -> torch.Tensor: diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/qwen_image.py b/python/sglang/multimodal_gen/configs/pipeline_configs/qwen_image.py index c99622e357..f57314ead6 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/qwen_image.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/qwen_image.py @@ -12,11 +12,15 @@ from sglang.multimodal_gen.configs.models.dits.qwenimage import ( QwenImageEditPlus_2511_DitConfig, ) from sglang.multimodal_gen.configs.models.encoders.qwen_image import Qwen2_5VLConfig +from sglang.multimodal_gen.configs.models.vaes.base import ( + get_channelwise_decode_scale_and_shift, +) from sglang.multimodal_gen.configs.models.vaes.qwenimage import QwenImageVAEConfig from sglang.multimodal_gen.configs.pipeline_configs.base import ( ImagePipelineConfig, ModelTaskType, maybe_unpad_latents, + pack_latents_2x2, pad_text_embeddings_with_mask, shard_rotary_emb_for_sp, ) @@ -141,19 +145,6 @@ def _shard_qwen_edit_freqs_cis_for_sp(freqs_cis, noisy_img_seq_len, device): ) -# Copied from diffusers.pipelines.qwenimage.pipeline_qwenimage.QwenImagePipeline._pack_latents -def _pack_latents(latents, batch_size, num_channels_latents, height, width): - latents = latents.view( - batch_size, num_channels_latents, height // 2, 2, width // 2, 2 - ) - latents = latents.permute(0, 2, 4, 1, 3, 5) - latents = latents.reshape( - batch_size, (height // 2) * (width // 2), num_channels_latents * 4 - ) - - return latents - - @dataclass class QwenImagePipelineConfig(QwenImageRolloutPipelineMixin, ImagePipelineConfig): """Configuration for the QwenImage pipeline.""" @@ -292,19 +283,14 @@ class QwenImagePipelineConfig(QwenImageRolloutPipelineMixin, ImagePipelineConfig width = 2 * (batch.width // (vae_scale_factor * 2)) num_channels_latents = self.dit_config.arch_config.in_channels // 4 # pack latents - return _pack_latents(latents, batch_size, num_channels_latents, height, width) + return pack_latents_2x2( + latents, batch_size, num_channels_latents, height, width + ) def get_decode_scale_and_shift(self, device, dtype, vae): - vae_arch_config = self.vae_config.arch_config - scaling_factor = 1.0 / torch.tensor( - vae_arch_config.latents_std, device=device - ).view(1, vae_arch_config.z_dim, 1, 1, 1).to(device, dtype) - shift_factor = ( - torch.tensor(vae_arch_config.latents_mean) - .view(1, vae_arch_config.z_dim, 1, 1, 1) - .to(device, dtype) + return get_channelwise_decode_scale_and_shift( + self.vae_config.arch_config, device, dtype ) - return scaling_factor, shift_factor @staticmethod def get_freqs_cis(img_shapes, txt_seq_lens, rotary_emb, device, dtype): @@ -594,7 +580,7 @@ class QwenImageEditPipelineConfig(QwenImagePipelineConfig): image_latents = latent_condition image_latent_height, image_latent_width = image_latents.shape[3:] num_channels_latents = self.dit_config.arch_config.in_channels // 4 - image_latents = _pack_latents( + image_latents = pack_latents_2x2( image_latents, batch_size, num_channels_latents, @@ -639,21 +625,6 @@ VAE_IMAGE_SIZE = 1024 * 1024 class QwenImageEditPlusPipelineConfig(QwenImageEditPipelineConfig): task_type: ModelTaskType = ModelTaskType.I2I - def _get_condition_image_sizes(self, batch) -> list[tuple[int, int]]: - image = batch.condition_image - if not isinstance(image, list): - image = [image] - - condition_image_sizes = [] - for img in image: - image_width, image_height = img.size - edit_width, edit_height, _ = _calculate_dimensions( - VAE_IMAGE_SIZE, image_width / image_height - ) - condition_image_sizes.append((edit_width, edit_height)) - - return condition_image_sizes - def prepare_image_processor_kwargs(self, batch, neg=False) -> dict: prompt = batch.prompt if not neg else batch.negative_prompt if not prompt: @@ -686,14 +657,6 @@ class QwenImageEditPlusPipelineConfig(QwenImageEditPipelineConfig): def prepare_calculated_size(self, image): return self.calculate_vae_image_size(image, image.width, image.height) - def resize_condition_image(self, images, target_width, target_height): - if not isinstance(images, list): - images = [images] - new_images = [] - for img, width, height in zip(images, target_width, target_height): - new_images.append(resize(img, height, width, resize_mode="default")) - return new_images - def calculate_condition_image_size(self, image, width, height) -> tuple[int, int]: calculated_width, calculated_height, _ = _calculate_dimensions( CONDITION_IMAGE_SIZE, width / height diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/sana_wm.py b/python/sglang/multimodal_gen/configs/pipeline_configs/sana_wm.py index 3dc674f6c7..ac32a230f4 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/sana_wm.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/sana_wm.py @@ -11,7 +11,10 @@ from sglang.multimodal_gen.configs.models.dits.sana_wm import SanaWMConfig from sglang.multimodal_gen.configs.models.encoders import BaseEncoderOutput from sglang.multimodal_gen.configs.models.encoders.base import EncoderConfig from sglang.multimodal_gen.configs.models.encoders.gemma2 import Gemma2Config -from sglang.multimodal_gen.configs.models.vaes.ltx_video import LTXVideoVAEConfig +from sglang.multimodal_gen.configs.models.vaes.ltx_video import ( + LTXVideoVAEConfig, + get_ltx_video_decode_scale_and_shift, +) from sglang.multimodal_gen.configs.pipeline_configs.base import ( ModelTaskType, PipelineConfig, @@ -286,42 +289,9 @@ class SanaWMPipelineConfig(PipelineConfig): return latents def get_decode_scale_and_shift(self, device, dtype, vae): - """Invert the LTX-2 latent normalization used before denoising. - - SANA-WM uses the LTX-2 VAE. Upstream encodes as - ``(z - latents_mean) * scaling_factor / latents_std`` and decodes by - applying the inverse transform. - """ - latents_mean = getattr(vae, "latents_mean", None) - latents_std = getattr(vae, "latents_std", None) - - scaling_factor = ( - getattr(getattr(vae, "config", None), "scaling_factor", None) - or getattr(vae, "scaling_factor", None) - or getattr(self.vae_config.arch_config, "scaling_factor", None) - or 1.0 + return get_ltx_video_decode_scale_and_shift( + device, dtype, vae, self.vae_config.arch_config ) - if isinstance(scaling_factor, (int, float)) and float(scaling_factor) == 0.0: - scaling_factor = 1.0 - - if isinstance(latents_mean, torch.Tensor) and isinstance( - latents_std, torch.Tensor - ): - latents_mean = latents_mean.to(device=device, dtype=dtype).view( - 1, -1, 1, 1, 1 - ) - latents_std = latents_std.to(device=device, dtype=dtype).view( - 1, -1, 1, 1, 1 - ) - sf = torch.tensor(float(scaling_factor), device=device, dtype=dtype).view( - 1, 1, 1, 1, 1 - ) - return sf / latents_std, latents_mean - - sf = torch.tensor(float(scaling_factor), device=device, dtype=dtype).view( - 1, 1, 1, 1, 1 - ) - return sf, None class SanaWMRealtimeConfig(SanaWMPipelineConfig): diff --git a/python/sglang/multimodal_gen/configs/sample/action.py b/python/sglang/multimodal_gen/configs/sample/action.py index c3f341516e..658fc2b356 100644 --- a/python/sglang/multimodal_gen/configs/sample/action.py +++ b/python/sglang/multimodal_gen/configs/sample/action.py @@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Any from sglang.multimodal_gen.configs.sample.sampling_params import ( DataType, + SamplingParams, _sanitize_filename, ) from sglang.multimodal_gen.configs.task_type import ModelTaskType @@ -65,15 +66,7 @@ class ActionSamplingParams: if env_steps is not None and self.num_inference_steps is not None: self.num_inference_steps = int(env_steps) - def build_request_extra(self) -> dict[str, Any]: - extra = {} - diffusers_kwargs = getattr(self, "diffusers_kwargs", None) - if diffusers_kwargs: - extra["diffusers_kwargs"] = diffusers_kwargs - explicit_fields = getattr(self, "_explicit_fields", None) - if explicit_fields is not None: - extra["explicit_fields"] = sorted(explicit_fields) - return extra + build_request_extra = SamplingParams.build_request_extra def apply_request_extra(self, req: Any) -> None: req.extra.update(self.build_request_extra()) diff --git a/python/sglang/multimodal_gen/configs/sample/minimax_h3.py b/python/sglang/multimodal_gen/configs/sample/minimax_h3.py index 6b18ebfcf7..f679e836f0 100644 --- a/python/sglang/multimodal_gen/configs/sample/minimax_h3.py +++ b/python/sglang/multimodal_gen/configs/sample/minimax_h3.py @@ -1,5 +1,4 @@ # SPDX-License-Identifier: Apache-2.0 -import math import os from collections.abc import Mapping from dataclasses import dataclass, field @@ -8,6 +7,7 @@ from typing import Any, ClassVar import msgspec from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams +from sglang.multimodal_gen.configs.utils import optional_positive_finite_float _MINIMAX_H3_MAX_SIGNED_SEED = (1 << 63) - 1 @@ -23,17 +23,6 @@ def _optional_unit_float(value: Any, field_name: str) -> float | None: return out -def _optional_positive_finite_float(value: Any, field_name: str) -> float | None: - if value is None: - return None - if isinstance(value, bool) or not isinstance(value, (int, float)): - raise ValueError(f"{field_name} must be a number") - out = float(value) - if not math.isfinite(out) or out <= 0.0: - raise ValueError(f"{field_name} must be a positive finite number") - return out - - @dataclass class MiniMaxH3SamplingParams(SamplingParams): # The video/audio sigma schedules include both interval endpoints. @@ -226,8 +215,8 @@ class MiniMaxH3SamplingParams(SamplingParams): if field_name in self.target } super()._validate() - _optional_positive_finite_float(self.flow_shift, "flow_shift") - _optional_positive_finite_float(self.audio_flow_shift, "audio_flow_shift") + optional_positive_finite_float(self.flow_shift, "flow_shift") + optional_positive_finite_float(self.audio_flow_shift, "audio_flow_shift") if self.enable_frame_interpolation: raise ValueError( "MiniMax H3 does not support enable_frame_interpolation: the " diff --git a/python/sglang/multimodal_gen/configs/sample/sampling_params.py b/python/sglang/multimodal_gen/configs/sample/sampling_params.py index 4159cf624f..51707c0477 100644 --- a/python/sglang/multimodal_gen/configs/sample/sampling_params.py +++ b/python/sglang/multimodal_gen/configs/sample/sampling_params.py @@ -758,10 +758,6 @@ class SamplingParams: RLRolloutArgs.validate_sampling_params(self) - def check_sampling_param(self): - # Keep backward-compatibility for old call sites. - self._validate() - def _validate_with_pipeline_config(self, pipeline_config): """ check if the sampling params is compatible and valid with server_args diff --git a/python/sglang/multimodal_gen/configs/utils.py b/python/sglang/multimodal_gen/configs/utils.py index 4388dccb7c..1084c6a6bb 100644 --- a/python/sglang/multimodal_gen/configs/utils.py +++ b/python/sglang/multimodal_gen/configs/utils.py @@ -1,12 +1,23 @@ # Copied and adapted from: https://github.com/hao-ai-lab/FastVideo -import argparse +import math import os from dataclasses import fields from operator import attrgetter from typing import Any +def optional_positive_finite_float(value: Any, field_name: str) -> float | None: + if value is None: + return None + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{field_name} must be a number") + out = float(value) + if not math.isfinite(out) or out <= 0.0: + raise ValueError(f"{field_name} must be a positive finite number") + return out + + def expand_path_fields(obj) -> None: """Expand paths in dataclass configuration without modifying other fields.""" for f in fields(obj): @@ -81,15 +92,3 @@ def update_config_from_args( args_dict.pop(key) return len(args_to_remove) > 0 - - -def clean_cli_args(args: argparse.Namespace) -> dict[str, Any]: - """ - Clean the arguments by removing the ones that not explicitly provided by the user. - """ - provided_args = {} - for k, v in vars(args).items(): - if v is not None and hasattr(args, "_provided") and k in args._provided: - provided_args[k] = v - - return provided_args diff --git a/python/sglang/multimodal_gen/envs.py b/python/sglang/multimodal_gen/envs.py index 4f0c8cfd0b..4d20d27861 100644 --- a/python/sglang/multimodal_gen/envs.py +++ b/python/sglang/multimodal_gen/envs.py @@ -5,12 +5,21 @@ import logging import os +import warnings from typing import TYPE_CHECKING, Any, Callable from sglang.multimodal_gen.runtime.utils.common import get_bool_env_var logger = logging.getLogger(__name__) +if "SGLANG_DIFFUSION_TRACE_FUNCTION" in os.environ: + warnings.warn( + "SGLANG_DIFFUSION_TRACE_FUNCTION was unused and has been removed. " + "Use Python profiling tools for function-level tracing.", + FutureWarning, + stacklevel=2, + ) + if TYPE_CHECKING: SGLANG_DIFFUSION_NCCL_SO_PATH: str | None = None LD_LIBRARY_PATH: str | None = None @@ -20,7 +29,6 @@ if TYPE_CHECKING: SGLANG_DIFFUSION_CONFIG_ROOT: str = os.path.expanduser("~/.config/sgl_diffusion") SGLANG_DIFFUSION_LOGGING_LEVEL: str = "INFO" SGLANG_DIFFUSION_LOGGING_PREFIX: str = "" - SGLANG_DIFFUSION_TRACE_FUNCTION: int = 0 SGLANG_DIFFUSION_DISABLE_EARLY_VAE_DECODER_CAST: bool = False SGLANG_DIFFUSION_DISABLE_VAE_DECODER_STORE: bool = False SGLANG_DIFFUSION_DISABLE_MAPPED_WILLNEED: bool = False @@ -229,10 +237,6 @@ environment_variables: dict[str, Callable[[], Any]] = { ), # if set, SGLANG_DIFFUSION_LOGGING_PREFIX will be prepended to all log messages "SGLANG_DIFFUSION_LOGGING_PREFIX": _lazy_str("SGLANG_DIFFUSION_LOGGING_PREFIX", ""), - # Trace function calls - # If set to 1, sgl_diffusion will trace function calls - # Useful for debugging - "SGLANG_DIFFUSION_TRACE_FUNCTION": _lazy_int("SGLANG_DIFFUSION_TRACE_FUNCTION", 0), # Path to the attention configuration file. Only used for sliding tile # attention for now. "SGLANG_DIFFUSION_ATTENTION_CONFIG": _lazy_path( diff --git a/python/sglang/multimodal_gen/runtime/breakable_cuda_graph/model_padders/ideogram.py b/python/sglang/multimodal_gen/runtime/breakable_cuda_graph/model_padders/ideogram.py index d4f7ec936a..104ba66161 100644 --- a/python/sglang/multimodal_gen/runtime/breakable_cuda_graph/model_padders/ideogram.py +++ b/python/sglang/multimodal_gen/runtime/breakable_cuda_graph/model_padders/ideogram.py @@ -30,16 +30,8 @@ def is_ideogram_transformer(current_model: Any, call_kwargs: dict) -> bool: ) -def _unwrap_model(current_model: Any) -> Any: - for attr in ("module", "_orig_mod"): - wrapped = getattr(current_model, attr, None) - if wrapped is not None: - current_model = wrapped - return current_model - - def _dynamic_mask_meta(current_model: Any) -> DynamicVarlenMaskMeta: - model = _unwrap_model(current_model) + model = bcg_utils.unwrap_model(current_model) meta = getattr(model, _DYNAMIC_MASK_META_ATTR, None) if not isinstance(meta, DynamicVarlenMaskMeta): meta = DynamicVarlenMaskMeta() diff --git a/python/sglang/multimodal_gen/runtime/breakable_cuda_graph/model_padders/zimage.py b/python/sglang/multimodal_gen/runtime/breakable_cuda_graph/model_padders/zimage.py index 7517ee3277..7e6134c47f 100644 --- a/python/sglang/multimodal_gen/runtime/breakable_cuda_graph/model_padders/zimage.py +++ b/python/sglang/multimodal_gen/runtime/breakable_cuda_graph/model_padders/zimage.py @@ -55,16 +55,8 @@ def _pad_caption(obj: Any, *, target: int) -> Any: return obj -def _unwrap_model(current_model: Any) -> Any: - for attr in ("module", "_orig_mod"): - wrapped = getattr(current_model, attr, None) - if wrapped is not None: - current_model = wrapped - return current_model - - def _build_caption_freqs(current_model: Any, *, target: int, device: torch.device): - rotary_emb = getattr(_unwrap_model(current_model), "rotary_emb", None) + rotary_emb = getattr(bcg_utils.unwrap_model(current_model), "rotary_emb", None) if rotary_emb is None: return None diff --git a/python/sglang/multimodal_gen/runtime/breakable_cuda_graph/prompt_padding.py b/python/sglang/multimodal_gen/runtime/breakable_cuda_graph/prompt_padding.py index e856bcc0c5..b9721d41dd 100644 --- a/python/sglang/multimodal_gen/runtime/breakable_cuda_graph/prompt_padding.py +++ b/python/sglang/multimodal_gen/runtime/breakable_cuda_graph/prompt_padding.py @@ -249,6 +249,14 @@ def pad_masked_prompt_kwargs(call_kwargs: dict, buckets: tuple[int, ...]) -> dic return out +def unwrap_model(current_model: Any) -> Any: + for attr in ("module", "_orig_mod"): + wrapped = getattr(current_model, attr, None) + if wrapped is not None: + current_model = wrapped + return current_model + + def transformer_class_name_matches(current_model: Any, needle: str) -> bool: """True when ``current_model`` (or its ``module`` / ``_orig_mod`` wrapper) is a transformer whose qualified class name contains ``needle``.""" diff --git a/python/sglang/multimodal_gen/runtime/disaggregation/disagg_args.py b/python/sglang/multimodal_gen/runtime/disaggregation/disagg_args.py deleted file mode 100644 index 5e6c35922a..0000000000 --- a/python/sglang/multimodal_gen/runtime/disaggregation/disagg_args.py +++ /dev/null @@ -1,28 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -"""Compatibility shim for disaggregated diffusion argument helpers.""" - -from __future__ import annotations - -import argparse - -from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType -from sglang.multimodal_gen.runtime.server_args.disagg import DisaggServerArgsMixin - -# Keep the historical disagg_args import path working. -DISAGG_RESULT_PORT_OFFSETS = DisaggServerArgsMixin.DISAGG_RESULT_PORT_OFFSETS -DisaggArgsMixin = DisaggServerArgsMixin - - -def add_disagg_cli_args(parser: argparse.ArgumentParser) -> None: - """Register disaggregated-diffusion CLI args through ServerArgs.""" - - from sglang.multimodal_gen.runtime.server_args import ServerArgs - - ServerArgs.add_disagg_cli_args(parser) - - -def convert_disagg_role_string(kwargs: dict) -> None: - """Convert ``disagg_role`` from string to ``RoleType`` enum in-place.""" - - if "disagg_role" in kwargs and isinstance(kwargs["disagg_role"], str): - kwargs["disagg_role"] = RoleType.from_string(kwargs["disagg_role"]) diff --git a/python/sglang/multimodal_gen/runtime/disaggregation/dispatch_policy.py b/python/sglang/multimodal_gen/runtime/disaggregation/dispatch_policy.py index 1382a5ef36..f39c97e3d0 100644 --- a/python/sglang/multimodal_gen/runtime/disaggregation/dispatch_policy.py +++ b/python/sglang/multimodal_gen/runtime/disaggregation/dispatch_policy.py @@ -2,11 +2,8 @@ """Dispatch policies for multi-instance disaggregated diffusion pipelines.""" import abc -import logging import threading -logger = logging.getLogger(__name__) - class DispatchPolicy(abc.ABC): def __init__(self, num_instances: int): @@ -19,16 +16,9 @@ class DispatchPolicy(abc.ABC): return self._num_instances @abc.abstractmethod - def select(self, active_counts: list[int] | None = None) -> int: ... - def select_with_capacity(self, free_slots: list[int]) -> int | None: """Select an instance that has free capacity, or None if all full.""" - if not any(s > 0 for s in free_slots): - return None - return self.select(active_counts=None) - - def record_completion(self, instance_id: int) -> None: - pass + ... class RoundRobin(DispatchPolicy): @@ -37,12 +27,6 @@ class RoundRobin(DispatchPolicy): self._lock = threading.Lock() self._next = 0 - def select(self, active_counts: list[int] | None = None) -> int: - with self._lock: - chosen = self._next - self._next = (self._next + 1) % self._num_instances - return chosen - def select_with_capacity(self, free_slots: list[int]) -> int | None: with self._lock: for _ in range(self._num_instances): @@ -56,43 +40,11 @@ class RoundRobin(DispatchPolicy): class MaxFreeSlotsFirst(DispatchPolicy): """Dispatch to the instance with the most free slots.""" - def __init__(self, num_instances: int, max_slots_per_instance: int = 1): + def __init__(self, num_instances: int): super().__init__(num_instances) - self._max_slots = max_slots_per_instance self._lock = threading.Lock() self._tiebreak = 0 - def select(self, active_counts: list[int] | None = None) -> int: - with self._lock: - if active_counts is None or len(active_counts) != self._num_instances: - chosen = self._tiebreak % self._num_instances - self._tiebreak += 1 - return chosen - - best_id = 0 - best_free = self._max_slots - active_counts[0] - for i in range(1, self._num_instances): - free = self._max_slots - active_counts[i] - if free > best_free: - best_free = free - best_id = i - elif free == best_free: - if i == (self._tiebreak % self._num_instances): - best_id = i - - self._tiebreak += 1 - - if best_free <= 0: - logger.warning( - "All %d instances are at capacity (%d slots each), " - "dispatching to instance %d anyway", - self._num_instances, - self._max_slots, - best_id, - ) - - return best_id - def select_with_capacity(self, free_slots: list[int]) -> int | None: with self._lock: best_id = -1 @@ -121,26 +73,10 @@ class PoolDispatcher: num_denoisers: int, num_decoders: int, policy_name: str = "round_robin", - **kwargs, ): - self.encoder_policy = create_dispatch_policy( - policy_name, num_encoders, **kwargs - ) - self.denoiser_policy = create_dispatch_policy( - policy_name, num_denoisers, **kwargs - ) - self.decoder_policy = create_dispatch_policy( - policy_name, num_decoders, **kwargs - ) - - def select_encoder(self, active_counts: list[int] | None = None) -> int: - return self.encoder_policy.select(active_counts) - - def select_denoiser(self, active_counts: list[int] | None = None) -> int: - return self.denoiser_policy.select(active_counts) - - def select_decoder(self, active_counts: list[int] | None = None) -> int: - return self.decoder_policy.select(active_counts) + self.encoder_policy = create_dispatch_policy(policy_name, num_encoders) + self.denoiser_policy = create_dispatch_policy(policy_name, num_denoisers) + self.decoder_policy = create_dispatch_policy(policy_name, num_decoders) def select_encoder_with_capacity(self, free_slots: list[int]) -> int | None: return self.encoder_policy.select_with_capacity(free_slots) @@ -152,7 +88,7 @@ class PoolDispatcher: return self.decoder_policy.select_with_capacity(free_slots) -def create_dispatch_policy(name: str, num_instances: int, **kwargs) -> DispatchPolicy: +def create_dispatch_policy(name: str, num_instances: int) -> DispatchPolicy: policies = { "round_robin": RoundRobin, "max_free_slots": MaxFreeSlotsFirst, @@ -162,4 +98,4 @@ def create_dispatch_policy(name: str, num_instances: int, **kwargs) -> DispatchP raise ValueError( f"Unknown dispatch policy '{name}'. Available: {list(policies.keys())}" ) - return cls(num_instances=num_instances, **kwargs) + return cls(num_instances=num_instances) diff --git a/python/sglang/multimodal_gen/runtime/disaggregation/metrics.py b/python/sglang/multimodal_gen/runtime/disaggregation/metrics.py index cfe0b9c706..b65d506a03 100644 --- a/python/sglang/multimodal_gen/runtime/disaggregation/metrics.py +++ b/python/sglang/multimodal_gen/runtime/disaggregation/metrics.py @@ -6,12 +6,6 @@ import time from dataclasses import dataclass -@dataclass -class _RequestTiming: - start_time: float - stage_start: float = 0.0 - - @dataclass class RoleStats: role: str @@ -54,7 +48,7 @@ class DisaggMetrics: self._failed = 0 self._timed_out = 0 - self._in_flight: dict[str, _RequestTiming] = {} + self._in_flight: dict[str, float] = {} self._last_latency = 0.0 self._max_latency = 0.0 @@ -71,14 +65,14 @@ class DisaggMetrics: def record_request_start(self, request_id: str) -> None: with self._lock: - self._in_flight[request_id] = _RequestTiming(start_time=time.monotonic()) + self._in_flight[request_id] = time.monotonic() def record_request_complete(self, request_id: str) -> None: now = time.monotonic() with self._lock: - timing = self._in_flight.pop(request_id, None) - if timing is not None: - latency = now - timing.start_time + start_time = self._in_flight.pop(request_id, None) + if start_time is not None: + latency = now - start_time self._last_latency = latency self._max_latency = max(self._max_latency, latency) self._total_latency += latency diff --git a/python/sglang/multimodal_gen/runtime/disaggregation/orchestrator.py b/python/sglang/multimodal_gen/runtime/disaggregation/orchestrator.py index fd3f85157e..3be3052097 100644 --- a/python/sglang/multimodal_gen/runtime/disaggregation/orchestrator.py +++ b/python/sglang/multimodal_gen/runtime/disaggregation/orchestrator.py @@ -865,8 +865,8 @@ class DiffusionServer: if idx is None: break entry = self._denoiser_tta.popleft() - self._transfer_dispatch_to_denoiser( - entry.request_id, entry.transfer_state, idx + self._transfer_dispatch_to_receiver( + entry.request_id, entry.transfer_state, idx, RoleType.DENOISER ) def _drain_decoder_tta(self) -> None: @@ -877,8 +877,8 @@ class DiffusionServer: if idx is None: break entry = self._decoder_tta.popleft() - self._transfer_dispatch_to_decoder( - entry.request_id, entry.transfer_state, idx + self._transfer_dispatch_to_receiver( + entry.request_id, entry.transfer_state, idx, RoleType.DECODER ) def _extract_request_id(self, frames: list) -> str | None: @@ -1161,36 +1161,57 @@ class DiffusionServer: ) receiver_pushes[receiver_idx].send_multipart(encode_transfer_msg(alloc_msg)) - def _transfer_dispatch_to_denoiser( - self, request_id: str, p2p: _TransferRequestState, denoiser_idx: int + def _transfer_dispatch_to_receiver( + self, + request_id: str, + p2p: _TransferRequestState, + receiver_idx: int, + role: RoleType, ) -> None: - self._denoiser_free_slots[denoiser_idx] -= 1 - p2p.receiver_instance = denoiser_idx + if role == RoleType.DENOISER: + free_slots = self._denoiser_free_slots + peers = self._denoiser_peers + sender_pushes = self._encoder_pushes + receiver_pushes = self._denoiser_pushes + source_role = "encoder" + running_state = RequestState.DENOISING_RUNNING + elif role == RoleType.DECODER: + free_slots = self._decoder_free_slots + peers = self._decoder_peers + sender_pushes = self._denoiser_pushes + receiver_pushes = self._decoder_pushes + source_role = "denoiser" + running_state = RequestState.DECODER_RUNNING + else: + raise ValueError(f"Invalid transfer receiver role: {role}") + + free_slots[receiver_idx] -= 1 + p2p.receiver_instance = receiver_idx try: self._tracker.transition( request_id, - RequestState.DENOISING_RUNNING, - denoiser_instance=denoiser_idx, + running_state, + **{f"{role.value}_instance": receiver_idx}, ) except ValueError: pass - peer_info = self._denoiser_peers.get(denoiser_idx, {}) + peer_info = peers.get(receiver_idx, {}) if not self._try_fast_path_push( request_id=request_id, p2p=p2p, receiver_peer_info=peer_info, - sender_pushes=self._encoder_pushes, - receiver_role_label="denoiser", - receiver_idx=denoiser_idx, + sender_pushes=sender_pushes, + receiver_role_label=role.value, + receiver_idx=receiver_idx, ): self._send_slow_path_alloc( request_id=request_id, p2p=p2p, - receiver_pushes=self._denoiser_pushes, - receiver_idx=denoiser_idx, - source_role="encoder", + receiver_pushes=receiver_pushes, + receiver_idx=receiver_idx, + source_role=source_role, ) def _handle_transfer_allocated(self, msg: dict) -> None: @@ -1378,38 +1399,6 @@ class DiffusionServer: self._transfer_state.pop(request_id, None) - def _transfer_dispatch_to_decoder( - self, request_id: str, p2p: _TransferRequestState, decoder_idx: int - ) -> None: - self._decoder_free_slots[decoder_idx] -= 1 - p2p.receiver_instance = decoder_idx - - try: - self._tracker.transition( - request_id, - RequestState.DECODER_RUNNING, - decoder_instance=decoder_idx, - ) - except ValueError: - pass - - peer_info = self._decoder_peers.get(decoder_idx, {}) - if not self._try_fast_path_push( - request_id=request_id, - p2p=p2p, - receiver_peer_info=peer_info, - sender_pushes=self._denoiser_pushes, - receiver_role_label="decoder", - receiver_idx=decoder_idx, - ): - self._send_slow_path_alloc( - request_id=request_id, - p2p=p2p, - receiver_pushes=self._decoder_pushes, - receiver_idx=decoder_idx, - source_role="denoiser", - ) - def _transfer_return_to_client_from_msg(self, request_id: str, msg: dict) -> None: with self._lock: client_identity = self._pending.pop(request_id, None) diff --git a/python/sglang/multimodal_gen/runtime/disaggregation/scheduler_mixin.py b/python/sglang/multimodal_gen/runtime/disaggregation/scheduler_mixin.py index 6a0867d393..834b7c8cd0 100644 --- a/python/sglang/multimodal_gen/runtime/disaggregation/scheduler_mixin.py +++ b/python/sglang/multimodal_gen/runtime/disaggregation/scheduler_mixin.py @@ -427,9 +427,7 @@ class SchedulerDisaggMixin: from sglang.multimodal_gen.runtime.disaggregation.metrics import DisaggMetrics self._disagg_role = server_args.disagg_role - self._disagg_timeout_s = float(getattr(server_args, "disagg_timeout", 600)) self._disagg_metrics = None - self._disagg_mode = getattr(server_args, "disagg_mode", False) self._pool_work_pull = None self._pool_result_push = None self._transfer_manager = None diff --git a/python/sglang/multimodal_gen/runtime/distributed/communication_op.py b/python/sglang/multimodal_gen/runtime/distributed/communication_op.py index e336e4cf8c..e9b267e216 100644 --- a/python/sglang/multimodal_gen/runtime/distributed/communication_op.py +++ b/python/sglang/multimodal_gen/runtime/distributed/communication_op.py @@ -45,11 +45,6 @@ def sequence_model_parallel_all_gather( return get_sp_group().all_gather(input_, dim) -def sequence_model_parallel_all_reduce(input_: torch.Tensor) -> torch.Tensor: - """All-reduce the input tensor across model parallel group.""" - return get_sp_group().all_reduce(input_) - - def cfg_model_parallel_all_gather( input_: torch.Tensor, dim: int = -1, separate_tensors: bool = False ) -> torch.Tensor: diff --git a/python/sglang/multimodal_gen/runtime/distributed/device_communicators/base_device_communicator.py b/python/sglang/multimodal_gen/runtime/distributed/device_communicators/base_device_communicator.py index c7a7dbdc08..b642071dbf 100644 --- a/python/sglang/multimodal_gen/runtime/distributed/device_communicators/base_device_communicator.py +++ b/python/sglang/multimodal_gen/runtime/distributed/device_communicators/base_device_communicator.py @@ -270,7 +270,6 @@ class DeviceCommunicatorBase: self.world_size = dist.get_world_size(cpu_group) self.ranks = dist.get_process_group_ranks(cpu_group) self.global_rank = dist.get_rank() - self.global_world_size = dist.get_world_size() self.rank_in_group = dist.get_group_rank(self.cpu_group, self.global_rank) def all_reduce( diff --git a/python/sglang/multimodal_gen/runtime/distributed/group_coordinator.py b/python/sglang/multimodal_gen/runtime/distributed/group_coordinator.py index a6fc8d9613..d8cf0c8a88 100644 --- a/python/sglang/multimodal_gen/runtime/distributed/group_coordinator.py +++ b/python/sglang/multimodal_gen/runtime/distributed/group_coordinator.py @@ -169,7 +169,6 @@ class GroupCoordinator: rank_in_group: int # rank inside the group cpu_group: ProcessGroup # group for CPU communication device_group: ProcessGroup # group for device communication - use_device_communicator: bool # whether to use device communicator device_communicator: DeviceCommunicatorBase # all_to_all_4D communicator def __init__( @@ -207,7 +206,6 @@ class GroupCoordinator: # TODO: fix it for other platforms self.device = get_local_torch_device() - self.use_device_communicator = use_device_communicator self.device_communicator: DeviceCommunicatorBase = None # type: ignore if use_device_communicator and self.world_size > 1: communicator_cls = _resolve_all_to_all_communicator_cls() @@ -228,10 +226,6 @@ class GroupCoordinator: # srt owns topology, dtype, contiguity, and size dispatch for custom ar self._init_srt_custom_allreduce() - # TODO(will): check if this is needed - # self.use_custom_op_call = current_platform.is_cuda_alike() - self.use_custom_op_call = False - def _init_srt_custom_allreduce(self) -> None: custom_allreduce_kwargs = { "group": self.cpu_group, diff --git a/python/sglang/multimodal_gen/runtime/distributed/parallel_state.py b/python/sglang/multimodal_gen/runtime/distributed/parallel_state.py index 7f76a04fce..826ae957fb 100644 --- a/python/sglang/multimodal_gen/runtime/distributed/parallel_state.py +++ b/python/sglang/multimodal_gen/runtime/distributed/parallel_state.py @@ -767,11 +767,6 @@ def get_sequence_parallel_world_size() -> int: return get_sp_world_size() -def get_sequence_parallel_rank() -> int: - """Return my rank for the sequence parallel group.""" - return get_sp_parallel_rank() - - def get_ulysses_parallel_world_size() -> int: return get_sp_group().ulysses_world_size @@ -803,32 +798,6 @@ def get_ring_ctx() -> tuple[int, int]: return get_ring_parallel_world_size(), get_ring_parallel_rank() -# PP -def get_pp_group() -> GroupCoordinator: - assert _PP is not None, "pipeline model parallel group is not initialized" - return _PP - - -def get_pipeline_parallel_world_size() -> int: - """Return world size for the pipeline model parallel group.""" - return get_pp_group().world_size - - -def get_pipeline_parallel_rank() -> int: - """Return my rank for the pipeline model parallel group.""" - return get_pp_group().rank_in_group - - -def is_pipeline_first_stage() -> bool: - """Return True if in the first pipeline model parallel stage, False otherwise.""" - return get_pipeline_parallel_rank() == 0 - - -def is_pipeline_last_stage() -> bool: - """Return True if in the last pipeline model parallel stage, False otherwise.""" - return get_pipeline_parallel_rank() == (get_pipeline_parallel_world_size() - 1) - - # CFG def get_cfg_group() -> GroupCoordinator: assert _CFG is not None, ( @@ -852,47 +821,6 @@ def get_data_parallel_world_size() -> int: return get_dp_world_size() -def get_data_parallel_rank() -> int: - """Return my rank for the data parallel group.""" - return get_dp_rank() - - -def is_dp_last_group() -> bool: - """Return True if in the last data parallel group, False otherwise.""" - return ( - get_sequence_parallel_rank() == (get_sequence_parallel_world_size() - 1) - and get_classifier_free_guidance_rank() - == (get_classifier_free_guidance_world_size() - 1) - and get_pipeline_parallel_rank() == (get_pipeline_parallel_world_size() - 1) - ) - - -def get_dit_world_size() -> int: - """Return world size for the DiT model (excluding VAE).""" - return ( - get_data_parallel_world_size() - * get_classifier_free_guidance_world_size() - * get_sequence_parallel_world_size() - * get_pipeline_parallel_world_size() - * get_tensor_model_parallel_world_size() - ) - - -def get_vae_parallel_group() -> ProcessGroup: - assert _VAE is not None, "VAE parallel group is not initialized" - return _VAE - - -def get_vae_parallel_world_size() -> int: - """Return world size for the VAE parallel group.""" - return torch.distributed.get_world_size(group=get_vae_parallel_group()) - - -def get_vae_parallel_rank() -> int: - """Return my rank for the VAE parallel group.""" - return torch.distributed.get_rank(group=get_vae_parallel_group()) - - def get_decode_parallel_group_coordinator() -> GroupCoordinator: assert _VAE_DECODE is not None, "VAE decode parallel group is not initialized" return _VAE_DECODE diff --git a/python/sglang/multimodal_gen/runtime/entrypoints/cli/generate.py b/python/sglang/multimodal_gen/runtime/entrypoints/cli/generate.py index dedcae21c7..3da67502c1 100644 --- a/python/sglang/multimodal_gen/runtime/entrypoints/cli/generate.py +++ b/python/sglang/multimodal_gen/runtime/entrypoints/cli/generate.py @@ -215,16 +215,6 @@ class GenerateSubcommand(CLISubcommand): def __init__(self) -> None: self.name = "generate" super().__init__() - self.init_arg_names = self._get_init_arg_names() - self.generation_arg_names = self._get_generation_arg_names() - - def _get_init_arg_names(self) -> list[str]: - """Get names of arguments for DiffGenerator initialization""" - return ["num_gpus", "tp_size", "sp_size", "model_path"] - - def _get_generation_arg_names(self) -> list[str]: - """Get names of arguments for generate_video method""" - return [field.name for field in dataclasses.fields(SamplingParams)] def cmd( self, args: argparse.Namespace, unknown_args: list[str] | None = None diff --git a/python/sglang/multimodal_gen/runtime/entrypoints/openai/protocol.py b/python/sglang/multimodal_gen/runtime/entrypoints/openai/protocol.py index 13e8990084..73bec055fd 100644 --- a/python/sglang/multimodal_gen/runtime/entrypoints/openai/protocol.py +++ b/python/sglang/multimodal_gen/runtime/entrypoints/openai/protocol.py @@ -173,10 +173,6 @@ class VideoListResponse(BaseModel): object: str = "list" -class VideoRemixRequest(BaseModel): - prompt: str - - class RealtimeVideoGenerationsRequest(VideoGenerationsRequest): enhance_prompt: Literal[False] = False type: Literal["init"] diff --git a/python/sglang/multimodal_gen/runtime/entrypoints/openai/realtime/adapters/sana_wm_realtime_adapter.py b/python/sglang/multimodal_gen/runtime/entrypoints/openai/realtime/adapters/sana_wm_realtime_adapter.py index dd8e409484..f29a2cbcd5 100644 --- a/python/sglang/multimodal_gen/runtime/entrypoints/openai/realtime/adapters/sana_wm_realtime_adapter.py +++ b/python/sglang/multimodal_gen/runtime/entrypoints/openai/realtime/adapters/sana_wm_realtime_adapter.py @@ -38,7 +38,6 @@ if TYPE_CHECKING: SANA_WM_DEFAULT_SIZE = "1280x704" -SANA_WM_DEFAULT_NUM_FRAMES = 1081 SANA_WM_DEFAULT_FPS = 16 SANA_WM_DEFAULT_STEPS = 4 SANA_WM_DEFAULT_GUIDANCE = 1.0 diff --git a/python/sglang/multimodal_gen/runtime/layers/activation.py b/python/sglang/multimodal_gen/runtime/layers/activation.py index e6a5357719..648057073c 100644 --- a/python/sglang/multimodal_gen/runtime/layers/activation.py +++ b/python/sglang/multimodal_gen/runtime/layers/activation.py @@ -41,9 +41,6 @@ class SiluAndMul(CustomOp): return: (num_tokens, d) or (batch_size, seq_len, d) """ - def __init__(self) -> None: - super().__init__() - def forward_cuda(self, x: torch.Tensor) -> torch.Tensor: d = x.shape[-1] // 2 output_shape = x.shape[:-1] + (d,) @@ -111,9 +108,6 @@ class GeluAndMul(CustomOp): @CustomOp.register("gelu_new") class NewGELU(CustomOp): - def __init__(self): - super().__init__() - def forward_cuda(self, *args, **kwargs) -> Any: return self.forward_native(*args, **kwargs) @@ -129,9 +123,6 @@ class NewGELU(CustomOp): @CustomOp.register("quick_gelu") class QuickGELU(CustomOp): # https://github.com/huggingface/transformers/blob/main/src/transformers/activations.py#L90 - def __init__(self): - super().__init__() - def forward_cuda(self, *args, **kwargs) -> Any: return self.forward_native(*args, **kwargs) diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/ascend_fa.py b/python/sglang/multimodal_gen/runtime/layers/attention/backends/ascend_fa.py index a9f88cfc48..01b819bd5d 100644 --- a/python/sglang/multimodal_gen/runtime/layers/attention/backends/ascend_fa.py +++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/ascend_fa.py @@ -196,12 +196,6 @@ class AscendFAMetadata: class AscendFAMetadataBuilder(AttentionMetadataBuilder): - def __init__(self) -> None: - pass - - def prepare(self) -> None: - pass - def build( self, **kwargs: dict[str, Any], diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/attention_backend.py b/python/sglang/multimodal_gen/runtime/layers/attention/backends/attention_backend.py index a0326d2dc5..b4626cbe26 100644 --- a/python/sglang/multimodal_gen/runtime/layers/attention/backends/attention_backend.py +++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/attention_backend.py @@ -95,16 +95,6 @@ T = TypeVar("T", bound=AttentionMetadata) class AttentionMetadataBuilder(ABC, Generic[T]): """Abstract class for attention metadata builders.""" - @abstractmethod - def __init__(self) -> None: - """Create the builder, remember some configuration and parameters.""" - raise NotImplementedError - - @abstractmethod - def prepare(self) -> None: - """Prepare for one batch.""" - raise NotImplementedError - @abstractmethod def build( self, diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/block_sparse_attn.py b/python/sglang/multimodal_gen/runtime/layers/attention/backends/block_sparse_attn.py index f51ffa9acc..419845c06c 100644 --- a/python/sglang/multimodal_gen/runtime/layers/attention/backends/block_sparse_attn.py +++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/block_sparse_attn.py @@ -53,12 +53,6 @@ class BlockSparseAttentionMetadata(AttentionMetadata): class BlockSparseAttentionMetadataBuilder(AttentionMetadataBuilder): - def __init__(self) -> None: - pass - - def prepare(self) -> None: - pass - def build( self, current_timestep: int, diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/cube_sparse_attn/backend.py b/python/sglang/multimodal_gen/runtime/layers/attention/backends/cube_sparse_attn/backend.py index 50c4d94ace..ef0b54786b 100644 --- a/python/sglang/multimodal_gen/runtime/layers/attention/backends/cube_sparse_attn/backend.py +++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/cube_sparse_attn/backend.py @@ -62,12 +62,6 @@ class CubeSparseAttentionMetadata(AttentionMetadata): class CubeSparseAttentionMetadataBuilder(AttentionMetadataBuilder): - def __init__(self): - pass - - def prepare(self): - pass - def build( # type: ignore[override] self, *, diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/flash_attn.py b/python/sglang/multimodal_gen/runtime/layers/attention/backends/flash_attn.py index 630677789a..0058204e46 100644 --- a/python/sglang/multimodal_gen/runtime/layers/attention/backends/flash_attn.py +++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/flash_attn.py @@ -322,12 +322,6 @@ class FlashAttentionMetadata: class FlashAttentionMetadataBuilder(AttentionMetadataBuilder): - def __init__(self) -> None: - pass - - def prepare(self) -> None: - pass - def build( # type: ignore self, raw_latent_shape=list, @@ -386,7 +380,6 @@ class FlashAttentionImpl(AttentionImpl): self.packed_trailing_padding = extra_impl_args.get( "packed_trailing_padding", False ) - self.attention_metadata = FlashAttentionMetadata() def _request_skip_softmax_threshold(self) -> tuple[bool, float | None]: params = get_request_skip_softmax_params() diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/hybrid_window_attn_h3.py b/python/sglang/multimodal_gen/runtime/layers/attention/backends/hybrid_window_attn_h3.py index 763e2eb067..f40d4d575c 100644 --- a/python/sglang/multimodal_gen/runtime/layers/attention/backends/hybrid_window_attn_h3.py +++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/hybrid_window_attn_h3.py @@ -268,12 +268,6 @@ class HybridWindowAttentionH3Metadata(AttentionMetadata): class HybridWindowAttentionH3MetadataBuilder(AttentionMetadataBuilder): - def __init__(self) -> None: - pass - - def prepare(self) -> None: - pass - def build( # type: ignore[override] self, *, diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/rain_fusion_attn.py b/python/sglang/multimodal_gen/runtime/layers/attention/backends/rain_fusion_attn.py index 48245874ea..583b07db8c 100644 --- a/python/sglang/multimodal_gen/runtime/layers/attention/backends/rain_fusion_attn.py +++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/rain_fusion_attn.py @@ -54,12 +54,6 @@ class RainFusionAttentionMetadata(AttentionMetadata): class RainFusionAttentionMetadataBuilder(AttentionMetadataBuilder): - def __init__(self) -> None: - pass - - def prepare(self) -> None: - pass - def build( self, current_timestep: int, diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/sliding_tile_attn.py b/python/sglang/multimodal_gen/runtime/layers/attention/backends/sliding_tile_attn.py index 17d7131a49..4c49773b39 100644 --- a/python/sglang/multimodal_gen/runtime/layers/attention/backends/sliding_tile_attn.py +++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/sliding_tile_attn.py @@ -80,12 +80,6 @@ class SlidingTileAttentionMetadata(AttentionMetadata): class SlidingTileAttentionMetadataBuilder(AttentionMetadataBuilder): - def __init__(self): - pass - - def prepare(self): - pass - def build( # type: ignore self, STA_param: list[list[Any]], diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/sparse_linear_attn.py b/python/sglang/multimodal_gen/runtime/layers/attention/backends/sparse_linear_attn.py index b3dcd6deec..827f7509b8 100644 --- a/python/sglang/multimodal_gen/runtime/layers/attention/backends/sparse_linear_attn.py +++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/sparse_linear_attn.py @@ -86,12 +86,6 @@ class SparseLinearAttentionMetadata(AttentionMetadata): class SparseLinearAttentionMetadataBuilder(AttentionMetadataBuilder): """Builder for SparseLinearAttentionMetadata.""" - def __init__(self) -> None: - pass - - def prepare(self) -> None: - pass - def build( self, current_timestep: int, @@ -104,6 +98,24 @@ class SparseLinearAttentionMetadataBuilder(AttentionMetadataBuilder): ) +def _initialize_linear_attention(model, head_size: int, feature_map: str) -> None: + model.proj_l = nn.Linear(head_size, head_size, dtype=torch.float32) + model.feature_map_q: Callable[[torch.Tensor], torch.Tensor] + model.feature_map_k: Callable[[torch.Tensor], torch.Tensor] + if feature_map == "elu": + model.feature_map_q = lambda x: F.elu(x) + 1 + model.feature_map_k = lambda x: F.elu(x) + 1 + elif feature_map == "relu": + model.feature_map_q = F.relu + model.feature_map_k = F.relu + elif feature_map == "softmax": + model.feature_map_q = lambda x: F.softmax(x, dim=-1) + model.feature_map_k = lambda x: F.softmax(x, dim=-1) + else: + raise ValueError(f"Unknown feature map: {feature_map}") + model._init_weights() + + class SparseLinearAttentionImpl(AttentionImpl, nn.Module): """Implementation of sparse linear attention for the backend.""" @@ -131,26 +143,7 @@ class SparseLinearAttentionImpl(AttentionImpl, nn.Module): self.BLKK = BLKK self.dtype = torch.bfloat16 if use_bf16 else torch.float16 - # Learnable linear projection for combining sparse + linear attention - self.proj_l = nn.Linear(head_size, head_size, dtype=torch.float32) - - # Feature map for linear attention - # Type annotation for callables - self.feature_map_q: Callable[[torch.Tensor], torch.Tensor] - self.feature_map_k: Callable[[torch.Tensor], torch.Tensor] - if feature_map == "elu": - self.feature_map_q = lambda x: F.elu(x) + 1 - self.feature_map_k = lambda x: F.elu(x) + 1 - elif feature_map == "relu": - self.feature_map_q = F.relu - self.feature_map_k = F.relu - elif feature_map == "softmax": - self.feature_map_q = lambda x: F.softmax(x, dim=-1) - self.feature_map_k = lambda x: F.softmax(x, dim=-1) - else: - raise ValueError(f"Unknown feature map: {feature_map}") - - self._init_weights() + _initialize_linear_attention(self, head_size, feature_map) def _init_weights(self) -> None: """Initialize projection weights to zero for residual-like behavior.""" @@ -322,12 +315,6 @@ class SageSparseLinearAttentionMetadata(AttentionMetadata): class SageSparseLinearAttentionMetadataBuilder(AttentionMetadataBuilder): """Builder for SageSparseLinearAttentionMetadata.""" - def __init__(self) -> None: - pass - - def prepare(self) -> None: - pass - def build( self, current_timestep: int, @@ -369,42 +356,12 @@ class SageSparseLinearAttentionImpl(AttentionImpl, nn.Module): self.topk_ratio = topk_ratio self.dtype = torch.bfloat16 if use_bf16 else torch.float16 - # Learnable linear projection for combining sparse + linear attention - self.proj_l = nn.Linear(head_size, head_size, dtype=torch.float32) + _initialize_linear_attention(self, head_size, feature_map) - # Feature map for linear attention - # Type annotation for callables - self.feature_map_q: Callable[[torch.Tensor], torch.Tensor] - self.feature_map_k: Callable[[torch.Tensor], torch.Tensor] - if feature_map == "elu": - self.feature_map_q = lambda x: F.elu(x) + 1 - self.feature_map_k = lambda x: F.elu(x) + 1 - elif feature_map == "relu": - self.feature_map_q = F.relu - self.feature_map_k = F.relu - elif feature_map == "softmax": - self.feature_map_q = lambda x: F.softmax(x, dim=-1) - self.feature_map_k = lambda x: F.softmax(x, dim=-1) - else: - raise ValueError(f"Unknown feature map: {feature_map}") - - self._init_weights() - - def _init_weights(self) -> None: - """Initialize projection weights to zero for residual-like behavior.""" - with torch.no_grad(): - nn.init.zeros_(self.proj_l.weight) - nn.init.zeros_(self.proj_l.bias) # type: ignore[arg-type] - - def _calc_linear_attention_with_torch( - self, - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - ): - kv = torch.matmul(k.transpose(-1, -2), v) - k_sum = torch.sum(k, dim=-2, keepdim=True) - return torch.matmul(q, kv) / (1e-5 + torch.matmul(q, k_sum.transpose(-1, -2))) + _init_weights = SparseLinearAttentionImpl._init_weights + _calc_linear_attention_with_torch = ( + SparseLinearAttentionImpl._calc_linear_attention_with_torch + ) def forward( self, diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/sparse_video_gen_2_attn.py b/python/sglang/multimodal_gen/runtime/layers/attention/backends/sparse_video_gen_2_attn.py index f5d03072c1..77493349e9 100644 --- a/python/sglang/multimodal_gen/runtime/layers/attention/backends/sparse_video_gen_2_attn.py +++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/sparse_video_gen_2_attn.py @@ -106,21 +106,7 @@ class SparseVideoGen2AttentionMetadata(AttentionMetadata): max_seqlen_k: int | None = None -def _require_kwarg(kwargs: dict[str, Any], name: str) -> Any: - if name not in kwargs: - raise ValueError( - f"Missing required argument for SparseVideoGen2Attention: {name}" - ) - return kwargs[name] - - class SparseVideoGen2AttentionMetadataBuilder(AttentionMetadataBuilder): - def __init__(self) -> None: - pass - - def prepare(self) -> None: - pass - def build( # type: ignore[override] self, current_timestep: int, diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/subblock_sparse_attn.py b/python/sglang/multimodal_gen/runtime/layers/attention/backends/subblock_sparse_attn.py index e441fdd1c1..58b791f456 100644 --- a/python/sglang/multimodal_gen/runtime/layers/attention/backends/subblock_sparse_attn.py +++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/subblock_sparse_attn.py @@ -397,14 +397,6 @@ class SubBlockSparseAttentionMetadata(AttentionMetadata): class SubBlockSparseAttentionMetadataBuilder(AttentionMetadataBuilder): - # The base class declares __init__ abstract, so a builder that does not - # override it cannot be instantiated at all. - def __init__(self) -> None: - pass - - def prepare(self) -> None: - pass - def build( # type: ignore[override] self, current_timestep: int, **kwargs: dict[str, Any] ) -> SubBlockSparseAttentionMetadata: diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/video_sparse_attn.py b/python/sglang/multimodal_gen/runtime/layers/attention/backends/video_sparse_attn.py index 6607321a82..6ae93233e2 100644 --- a/python/sglang/multimodal_gen/runtime/layers/attention/backends/video_sparse_attn.py +++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/video_sparse_attn.py @@ -174,12 +174,6 @@ def _compute_cur_topk(attn_metadata: VideoSparseAttentionMetadata) -> int: class VideoSparseAttentionMetadataBuilder(AttentionMetadataBuilder): - def __init__(self): - pass - - def prepare(self): - pass - def build( # type: ignore self, current_timestep: int, @@ -189,7 +183,6 @@ class VideoSparseAttentionMetadataBuilder(AttentionMetadataBuilder): device: torch.device, **kwargs: dict[str, Any], ) -> VideoSparseAttentionMetadata: - patch_size = patch_size dit_seq_shape = ( raw_latent_shape[0] // patch_size[0], raw_latent_shape[1] // patch_size[1], diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/video_sparse_attn_h3.py b/python/sglang/multimodal_gen/runtime/layers/attention/backends/video_sparse_attn_h3.py index aefb70f046..332c1d04a8 100644 --- a/python/sglang/multimodal_gen/runtime/layers/attention/backends/video_sparse_attn_h3.py +++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/video_sparse_attn_h3.py @@ -175,9 +175,6 @@ class VideoSparseAttentionH3MetadataBuilder(AttentionMetadataBuilder): def __init__(self) -> None: self._workspace_cache: dict = {} - def prepare(self) -> None: - pass - def build( # type: ignore[override] self, current_timestep: int, diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/vmoba.py b/python/sglang/multimodal_gen/runtime/layers/attention/backends/vmoba.py index 614c383076..193f228e58 100644 --- a/python/sglang/multimodal_gen/runtime/layers/attention/backends/vmoba.py +++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/vmoba.py @@ -88,12 +88,6 @@ def pad_input(hidden_states, indices, batch, seqlen): class VideoMobaAttentionMetadataBuilder(AttentionMetadataBuilder): - def __init__(self): - pass - - def prepare(self): - pass - def build( # type: ignore self, current_timestep: int, diff --git a/python/sglang/multimodal_gen/runtime/layers/custom_op.py b/python/sglang/multimodal_gen/runtime/layers/custom_op.py index 0fa1c3fb1c..88914ec549 100644 --- a/python/sglang/multimodal_gen/runtime/layers/custom_op.py +++ b/python/sglang/multimodal_gen/runtime/layers/custom_op.py @@ -129,14 +129,6 @@ class CustomOp(nn.Module): # since we are not using Inductor, we always return True return True - @staticmethod - def default_on() -> bool: - """ - On by default if level < CompilationLevel.PIECEWISE - Specifying 'all' or 'none' in custom_op takes precedence. - """ - raise NotImplementedError - # Dictionary of all custom ops (classes, indexed by registered name). # To check if an op with a name is enabled, call .enabled() on the class. # Examples: diff --git a/python/sglang/multimodal_gen/runtime/layers/linear.py b/python/sglang/multimodal_gen/runtime/layers/linear.py index ec8ae6b68a..204d412644 100644 --- a/python/sglang/multimodal_gen/runtime/layers/linear.py +++ b/python/sglang/multimodal_gen/runtime/layers/linear.py @@ -88,6 +88,27 @@ def adjust_scalar_to_fused_array( return param[shard_id], loaded_weight +def _load_tensor_parallel_weight( + param: Parameter, + loaded_weight: torch.Tensor, + tp_rank: int, + shard_dim: int | None, +) -> None: + is_sharded_weight = getattr(param, "is_sharded_weight", False) + param_data = param.data + # Already-sharded checkpoints (e.g. bitsandbytes) must not be sliced twice. + if shard_dim is not None and not is_sharded_weight: + shard_size = param_data.shape[shard_dim] + loaded_weight = loaded_weight.narrow( + shard_dim, tp_rank * shard_size, shard_size + ) + # AutoFP8 scales can be stored as scalars rather than length-one tensors. + if len(loaded_weight.shape) == 0: + loaded_weight = loaded_weight.reshape(1) + assert param_data.shape == loaded_weight.shape + param_data.copy_(loaded_weight) + + class LinearMethodBase(QuantizeMethodBase): """Base class for different (maybe quantized) linear methods.""" @@ -477,25 +498,9 @@ class ColumnParallelLinear(LinearBase): self.register_parameter("bias", None) def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor) -> None: - tp_rank = self.tp_rank - output_dim = getattr(param, "output_dim", None) - - is_sharded_weight = getattr(param, "is_sharded_weight", False) - is_sharded_weight = is_sharded_weight - - param_data = param.data - if output_dim is not None and not is_sharded_weight: - shard_size = param_data.shape[output_dim] - start_idx = tp_rank * shard_size - loaded_weight = loaded_weight.narrow(output_dim, start_idx, shard_size) - - # Special case for loading scales off disk, which often do not - # have a shape (such as in the case of AutoFP8). - if len(loaded_weight.shape) == 0: - loaded_weight = loaded_weight.reshape(1) - - assert param_data.shape == loaded_weight.shape - param_data.copy_(loaded_weight) + _load_tensor_parallel_weight( + param, loaded_weight, self.tp_rank, getattr(param, "output_dim", None) + ) def weight_loader_v2(self, param: Parameter, loaded_weight: torch.Tensor) -> None: # Special case for loading scales off disk, which often do not @@ -629,7 +634,6 @@ class MergedColumnParallelLinear(ColumnParallelLinear): is_sharded_weight = getattr(param, "is_sharded_weight", False) # bitsandbytes loads the weights of the specific portion # no need to narrow - is_sharded_weight = is_sharded_weight param_data = param_data.narrow(output_dim, shard_offset, shard_size) start_idx = tp_rank * shard_size @@ -1027,7 +1031,6 @@ class QKVParallelLinear(ColumnParallelLinear): is_sharded_weight = getattr(param, "is_sharded_weight", False) # bitsandbytes loads the weights of the specific portion # no need to narrow - is_sharded_weight = is_sharded_weight shard_idx = 0 param_data = param_data.narrow(output_dim, shard_offset, shard_size) @@ -1151,26 +1154,9 @@ class RowParallelLinear(LinearBase): self.register_parameter("bias", None) def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor): - tp_rank = self.tp_rank - input_dim = getattr(param, "input_dim", None) - is_sharded_weight = getattr(param, "is_sharded_weight", False) - # bitsandbytes loads the weights of the specific portion - # no need to narrow - is_sharded_weight = is_sharded_weight - - param_data = param.data - if input_dim is not None and not is_sharded_weight: - shard_size = param_data.shape[input_dim] - start_idx = tp_rank * shard_size - loaded_weight = loaded_weight.narrow(input_dim, start_idx, shard_size) - - # Special case for loading scales off disk, which often do not - # have a shape (such as in the case of AutoFP8). - if len(loaded_weight.shape) == 0: - loaded_weight = loaded_weight.reshape(1) - - assert param_data.shape == loaded_weight.shape - param_data.copy_(loaded_weight) + _load_tensor_parallel_weight( + param, loaded_weight, self.tp_rank, getattr(param, "input_dim", None) + ) def weight_loader_v2(self, param: BasevLLMParameter, loaded_weight: torch.Tensor): # Special case for loading scales off disk, which often do not diff --git a/python/sglang/multimodal_gen/runtime/layers/lora/linear.py b/python/sglang/multimodal_gen/runtime/layers/lora/linear.py index b8b3d4e749..1dca740132 100644 --- a/python/sglang/multimodal_gen/runtime/layers/lora/linear.py +++ b/python/sglang/multimodal_gen/runtime/layers/lora/linear.py @@ -32,9 +32,6 @@ from sglang.multimodal_gen.runtime.layers.linear import ( from sglang.multimodal_gen.runtime.layers.quantization.weight_only_fp8 import ( WeightOnlyFP8Linear, ) -from sglang.multimodal_gen.runtime.layers.vocab_parallel_embedding import ( - VocabParallelEmbedding, -) from sglang.multimodal_gen.runtime.managers.forward_context import ( get_forward_context_or_none, ) @@ -651,37 +648,7 @@ class BaseLayerWithLoRA(nn.Module): self.strength = 1.0 -class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA): - """ - Vocab parallel embedding layer with support for LoRA (Low-Rank Adaptation). - - Note: The current version does not yet implement the LoRA functionality. - This class behaves exactly the same as the base VocabParallelEmbedding. - Future versions will integrate LoRA functionality to support efficient parameter fine-tuning. - """ - - def __init__( - self, - base_layer: VocabParallelEmbedding, - ) -> None: - super().__init__(base_layer) - - def forward(self, input_: torch.Tensor) -> torch.Tensor: - raise NotImplementedError( - "We don't support VocabParallelEmbeddingWithLoRA yet." - ) - - class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA): - def __init__( - self, - base_layer: ColumnParallelLinear, - lora_rank: int | None = None, - lora_alpha: int | None = None, - snapshot_base: bool = True, - ) -> None: - super().__init__(base_layer, lora_rank, lora_alpha, snapshot_base) - def forward(self, input_: torch.Tensor) -> torch.Tensor: if self.disable_lora or (self.merged and not self.has_lora_output_offset): return self.base_layer(input_) @@ -738,15 +705,6 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA): class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA): - def __init__( - self, - base_layer: MergedColumnParallelLinear, - lora_rank: int | None = None, - lora_alpha: int | None = None, - snapshot_base: bool = True, - ) -> None: - super().__init__(base_layer, lora_rank, lora_alpha, snapshot_base) - def slice_lora_b_weights(self, B: torch.Tensor) -> torch.Tensor: tp_rank = get_tp_rank() if B.dim() == 3: @@ -772,15 +730,6 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA): class QKVParallelLinearWithLoRA(ColumnParallelLinearWithLoRA): - def __init__( - self, - base_layer: QKVParallelLinear, - lora_rank: int | None = None, - lora_alpha: int | None = None, - snapshot_base: bool = True, - ) -> None: - super().__init__(base_layer, lora_rank, lora_alpha, snapshot_base) - def slice_lora_b_weights( self, B: list[torch.Tensor] ) -> tuple[torch.Tensor, torch.Tensor]: @@ -802,15 +751,6 @@ class QKVParallelLinearWithLoRA(ColumnParallelLinearWithLoRA): class RowParallelLinearWithLoRA(BaseLayerWithLoRA): - def __init__( - self, - base_layer: RowParallelLinear, - lora_rank: int | None = None, - lora_alpha: int | None = None, - snapshot_base: bool = True, - ) -> None: - super().__init__(base_layer, lora_rank, lora_alpha, snapshot_base) - def forward(self, input_: torch.Tensor): if self.disable_lora or (self.merged and not self.has_lora_output_offset): return self.base_layer(input_) @@ -890,15 +830,6 @@ class LinearWithLoRA(BaseLayerWithLoRA): not a tuple of (output, bias). """ - def __init__( - self, - base_layer: nn.Linear, - lora_rank: int | None = None, - lora_alpha: int | None = None, - snapshot_base: bool = True, - ) -> None: - super().__init__(base_layer, lora_rank, lora_alpha, snapshot_base) - @torch.compile() def forward(self, x: torch.Tensor) -> torch.Tensor: lora_A = self.lora_A @@ -989,7 +920,6 @@ def wrap_with_lora_layer( """ supported_layer_types: dict[type[nn.Module], type[BaseLayerWithLoRA]] = { # the order matters - # VocabParallelEmbedding: VocabParallelEmbeddingWithLoRA, # Weight-only FP8 LoRA is currently dynamic-only and intended for # single-GPU deployments. WeightOnlyFP8Linear: WeightOnlyFP8LinearWithLoRA, diff --git a/python/sglang/multimodal_gen/runtime/layers/parallel_conv.py b/python/sglang/multimodal_gen/runtime/layers/parallel_conv.py index f1b45d781a..3b2081566c 100644 --- a/python/sglang/multimodal_gen/runtime/layers/parallel_conv.py +++ b/python/sglang/multimodal_gen/runtime/layers/parallel_conv.py @@ -181,7 +181,6 @@ def split_height_for_parallel_decode( def _maybe_contiguous_for_sp_gather(x: torch.Tensor) -> torch.Tensor: if ( x.dim() == 5 - and hasattr(torch, "channels_last_3d") and x.is_contiguous(memory_format=torch.channels_last_3d) and not x.is_contiguous() ): @@ -302,7 +301,7 @@ def gather_variable_height( def _halo_memory_format(reference: torch.Tensor) -> torch.memory_format: if reference.dim() > 1 and reference.stride(1) == 1: - if reference.dim() == 5 and hasattr(torch, "channels_last_3d"): + if reference.dim() == 5: return torch.channels_last_3d if reference.dim() == 4: return torch.channels_last @@ -475,7 +474,6 @@ def _conv_preserves_local_height( def _conv3d_weight_is_channels_last_3d(weight: torch.Tensor) -> bool: return ( weight.dim() == 5 - and hasattr(torch, "channels_last_3d") and (current_platform.is_cuda() or current_platform.is_rocm()) and weight.is_contiguous(memory_format=torch.channels_last_3d) ) diff --git a/python/sglang/multimodal_gen/runtime/layers/quantization/convrot_int8_comfy_kitchen.py b/python/sglang/multimodal_gen/runtime/layers/quantization/convrot_int8_comfy_kitchen.py index b33a9c1eb4..435893910c 100644 --- a/python/sglang/multimodal_gen/runtime/layers/quantization/convrot_int8_comfy_kitchen.py +++ b/python/sglang/multimodal_gen/runtime/layers/quantization/convrot_int8_comfy_kitchen.py @@ -21,7 +21,10 @@ import os import torch from torch.nn.parameter import Parameter -from sglang.multimodal_gen.runtime.layers.linear import LinearMethodBase +from sglang.multimodal_gen.runtime.layers.linear import ( + LinearMethodBase, + UnquantizedLinearMethod, +) from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import ( ConvRotInt8Config, ) @@ -118,20 +121,17 @@ class ConvRotInt8ComfyKitchenLinearMethod(LinearMethodBase): "--quantization-ignored-layers" ) - # The online path initially matches UnquantizedLinearMethod so the - # source weights load in BF16 before quantization. Serialized weights - # allocate their final INT8 storage immediately. - weight = Parameter( - torch.empty( - sum(output_partition_sizes), - input_size_per_partition, - dtype=(torch.int8 if self.is_checkpoint_serialized else params_dtype), - ), - requires_grad=False, + # Online checkpoints load their original dtype; serialized ones use INT8. + UnquantizedLinearMethod.create_weights( + self, + layer, + input_size_per_partition, + output_partition_sizes, + input_size, + output_size, + torch.int8 if self.is_checkpoint_serialized else params_dtype, + **extra_weight_attrs, ) - set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0}) - layer.register_parameter("weight", weight) - set_weight_attrs(weight, extra_weight_attrs) if self.is_checkpoint_serialized: weight_scale = Parameter( torch.empty( diff --git a/python/sglang/multimodal_gen/runtime/layers/quantization/convrot_int8_jit.py b/python/sglang/multimodal_gen/runtime/layers/quantization/convrot_int8_jit.py index 5e4469246f..5831c3884d 100644 --- a/python/sglang/multimodal_gen/runtime/layers/quantization/convrot_int8_jit.py +++ b/python/sglang/multimodal_gen/runtime/layers/quantization/convrot_int8_jit.py @@ -43,6 +43,7 @@ from sglang.multimodal_gen.runtime.layers.linear import ( LinearMethodBase, ReplicatedLinear, RowParallelLinear, + UnquantizedLinearMethod, ) from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import ( ConvRotInt8Config, @@ -202,20 +203,17 @@ class ConvRotInt8JitLinearMethod(LinearMethodBase): f"convrot_int8 backend jit needs the output size per " f"partition ({sum(output_partition_sizes)}) to be a multiple of 8" ) - # The online path initially matches UnquantizedLinearMethod so the - # source weights load in BF16 before quantization. Serialized weights - # allocate their final INT8 storage immediately. - weight = Parameter( - torch.empty( - sum(output_partition_sizes), - input_size_per_partition, - dtype=(torch.int8 if self.is_checkpoint_serialized else params_dtype), - ), - requires_grad=False, + # Online checkpoints load their original dtype; serialized ones use INT8. + UnquantizedLinearMethod.create_weights( + self, + layer, + input_size_per_partition, + output_partition_sizes, + input_size, + output_size, + torch.int8 if self.is_checkpoint_serialized else params_dtype, + **extra_weight_attrs, ) - set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0}) - layer.register_parameter("weight", weight) - set_weight_attrs(weight, extra_weight_attrs) if self.is_checkpoint_serialized: # Comfy stores the per-row scale as [N, 1]; the ops read N contiguous # floats and accept that shape unchanged. diff --git a/python/sglang/multimodal_gen/runtime/layers/quantization/modelopt_quant.py b/python/sglang/multimodal_gen/runtime/layers/quantization/modelopt_quant.py index 1effce239a..2e784e31c6 100755 --- a/python/sglang/multimodal_gen/runtime/layers/quantization/modelopt_quant.py +++ b/python/sglang/multimodal_gen/runtime/layers/quantization/modelopt_quant.py @@ -21,6 +21,9 @@ from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config impor from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import ( ConvRotInt8Config, ) +from sglang.multimodal_gen.runtime.layers.quantization.modelopt_fp8 import ( + ModelOptFp8LinearMethod as StaticModelOptFp8LinearMethod, +) from sglang.multimodal_gen.runtime.models.parameter import ( ModelWeightParameter, PerTensorScaleParameter, @@ -28,7 +31,6 @@ from sglang.multimodal_gen.runtime.models.parameter import ( from sglang.multimodal_gen.runtime.platforms import current_platform from sglang.multimodal_gen.runtime.utils.weight_attrs import set_weight_attrs from sglang.srt.layers.quantization.fp8_utils import ( - apply_fp8_linear, cutlass_fp8_supported, normalize_e4m3fn_to_e4m3fnuz, ) @@ -578,20 +580,7 @@ class ModelOptFp8LinearMethod(LinearMethodBase): copy_or_rebind_param(layer, "weight_scale", processed_weight_scale) copy_or_rebind_param(layer, "input_scale", layer.input_scale.max()) - def apply( - self, - layer: torch.nn.Module, - x: torch.Tensor, - bias: Optional[torch.Tensor] = None, - ) -> torch.Tensor: - return apply_fp8_linear( - input=x, - weight=layer.weight, - weight_scale=layer.weight_scale, - input_scale=layer.input_scale, - bias=bias, - cutlass_fp8_supported=self.cutlass_fp8_supported, - ) + apply = StaticModelOptFp8LinearMethod.apply class ModelOptFp4LinearMethod(LinearMethodBase): @@ -820,33 +809,9 @@ class ModelOptFp4LinearMethod(LinearMethodBase): ) x_fp4, x_scale_interleaved = fp4_quantize(x, layer.input_scale_inv) - weights_padding_cols = getattr(layer, "weights_padding_cols", 0) - x_fp4 = pad_nvfp4_activation_for_cutlass(x_fp4, weights_padding_cols) - - w = layer.weight - w_scale_interleaved = layer.weight_scale_interleaved - - if x_scale_interleaved.dtype == torch.uint8: - x_scale_interleaved = x_scale_interleaved.view(torch.float8_e4m3fn) - if w_scale_interleaved.dtype == torch.uint8: - w_scale_interleaved = w_scale_interleaved.view(torch.float8_e4m3fn) - fp4_gemm, flashinfer_backend = _get_fp4_gemm_op() - if fp4_gemm is None: - raise RuntimeError("No FP4 GEMM kernel available. Install flashinfer.") - out = fp4_gemm( - x_fp4, - w.T, - x_scale_interleaved, - w_scale_interleaved.T, - layer.alpha, - output_dtype, - backend=flashinfer_backend, + out = apply_nvfp4_gemm_prequantized( + layer, x_fp4, x_scale_interleaved, output_dtype, bias ) - - out = slice_nvfp4_output(out, output_size) - - if bias is not None: - out = out + bias return out.view(*output_shape) diff --git a/python/sglang/multimodal_gen/runtime/layers/quantization/mxfp4_npu.py b/python/sglang/multimodal_gen/runtime/layers/quantization/mxfp4_npu.py index 7cb9bd3118..f8c8cb395d 100644 --- a/python/sglang/multimodal_gen/runtime/layers/quantization/mxfp4_npu.py +++ b/python/sglang/multimodal_gen/runtime/layers/quantization/mxfp4_npu.py @@ -44,9 +44,6 @@ logger = init_logger(__name__) class NPUMXFP4Config(QuantizationConfig): """Config for online MXFP4 quantization on NPU (Diffusion).""" - def __init__(self) -> None: - super().__init__() - @classmethod def get_name(cls) -> str: return "mxfp4_npu" diff --git a/python/sglang/multimodal_gen/runtime/layers/rotary_embedding/factory.py b/python/sglang/multimodal_gen/runtime/layers/rotary_embedding/factory.py index f039782647..fcffe9e073 100644 --- a/python/sglang/multimodal_gen/runtime/layers/rotary_embedding/factory.py +++ b/python/sglang/multimodal_gen/runtime/layers/rotary_embedding/factory.py @@ -10,7 +10,6 @@ from .mrope import NDRotaryEmbedding, _to_tuple _ROPE_DICT: dict[tuple, RotaryEmbedding] = {} _ND_ROPE_CACHE: "OrderedDict[tuple, NDRotaryEmbedding]" = OrderedDict() -_ROPE_3D_CACHE: "OrderedDict[tuple, tuple[torch.Tensor, torch.Tensor]]" = OrderedDict() def get_rope( diff --git a/python/sglang/multimodal_gen/runtime/layers/rotary_embedding/mrope.py b/python/sglang/multimodal_gen/runtime/layers/rotary_embedding/mrope.py index 2981393bf0..c6b738b131 100644 --- a/python/sglang/multimodal_gen/runtime/layers/rotary_embedding/mrope.py +++ b/python/sglang/multimodal_gen/runtime/layers/rotary_embedding/mrope.py @@ -367,6 +367,26 @@ class NDRotaryEmbedding(torch.nn.Module): positions = torch.tensor(pos_tuple, dtype=torch.long, device=device) return self.forward_uncached(pos=positions) + def forward_3d_sequence_shard( + self, + local_len: int, + rank: int, + frame_stride_local: int, + width_local: int, + device: torch.device, + ) -> tuple[torch.Tensor, torch.Tensor]: + """Embed a contiguous shard of a flattened (time, height, width) grid.""" + token_start = rank * local_len + token_indices = torch.arange( + token_start, token_start + local_len, device=device, dtype=torch.long + ) + t_idx = token_indices // frame_stride_local + rem = token_indices % frame_stride_local + h_idx = rem // width_local + w_idx = rem % width_local + positions = torch.stack((t_idx, h_idx, w_idx), dim=1) + return self.forward_uncached(positions) + def forward_uncached(self, pos: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: """ The core implementation that computes embeddings from a position tensor. diff --git a/python/sglang/multimodal_gen/runtime/layers/usp.py b/python/sglang/multimodal_gen/runtime/layers/usp.py index 487a8874ff..6be91d9404 100644 --- a/python/sglang/multimodal_gen/runtime/layers/usp.py +++ b/python/sglang/multimodal_gen/runtime/layers/usp.py @@ -6,7 +6,6 @@ from typing import TYPE_CHECKING import torch import torch.distributed as dist -import torch.distributed._functional_collectives as ft_c from torch.distributed.tensor.experimental._attention import _cp_options from sglang.kernels.ops.diffusion import pack_qkv_destination_major, usp_merge_heads @@ -28,16 +27,6 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -def _maybe_wait(tensor: torch.Tensor) -> torch.Tensor: - """ - When tracing the code, the result tensor is not an AsyncCollectiveTensor, - so we cannot call ``wait()``. - """ - if isinstance(tensor, ft_c.AsyncCollectiveTensor): - return tensor.wait() - return tensor - - _A2A_STAGING_BUFFERS: dict[tuple[str, torch.dtype, int], torch.Tensor] = {} diff --git a/python/sglang/multimodal_gen/runtime/layers/utils.py b/python/sglang/multimodal_gen/runtime/layers/utils.py index 58e2416ca8..291c2005e1 100644 --- a/python/sglang/multimodal_gen/runtime/layers/utils.py +++ b/python/sglang/multimodal_gen/runtime/layers/utils.py @@ -79,13 +79,7 @@ def direct_register_custom_op( # Operator doesn't exist, proceed with registration pass - if hasattr(torch.library, "infer_schema"): - schema_str = torch.library.infer_schema(op_func, mutates_args=mutates_args) - else: - # for pytorch 2.4 - import torch._custom_op.impl - - schema_str = torch._custom_op.impl.infer_schema(op_func, mutates_args) + schema_str = torch.library.infer_schema(op_func, mutates_args=mutates_args) try: my_lib.define(op_name + schema_str) @@ -106,9 +100,6 @@ def direct_register_custom_op( else: # Re-raise other RuntimeErrors raise error - except AttributeError as error: - # Always re-raise AttributeError as it indicates missing dependencies - raise error class CustomOpWrapper: diff --git a/python/sglang/multimodal_gen/runtime/layers/vocab_parallel_embedding.py b/python/sglang/multimodal_gen/runtime/layers/vocab_parallel_embedding.py index 050f9e9aca..565c2f77f6 100644 --- a/python/sglang/multimodal_gen/runtime/layers/vocab_parallel_embedding.py +++ b/python/sglang/multimodal_gen/runtime/layers/vocab_parallel_embedding.py @@ -15,6 +15,7 @@ from sglang.multimodal_gen.runtime.distributed import ( get_tp_group, tensor_model_parallel_all_reduce, ) +from sglang.multimodal_gen.runtime.layers.linear import UnquantizedLinearMethod from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config import ( QuantizationConfig, QuantizeMethodBase, @@ -23,7 +24,6 @@ from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config impor from sglang.multimodal_gen.runtime.layers.utils import get_group_rank, get_group_size from sglang.multimodal_gen.runtime.models.parameter import BasevLLMParameter from sglang.multimodal_gen.runtime.platforms import current_platform -from sglang.multimodal_gen.runtime.utils.weight_attrs import set_weight_attrs DEFAULT_VOCAB_PADDING_SIZE = 64 @@ -31,29 +31,7 @@ DEFAULT_VOCAB_PADDING_SIZE = 64 class UnquantizedEmbeddingMethod(QuantizeMethodBase): """Unquantized method for embeddings.""" - def create_weights( - self, - layer: torch.nn.Module, - input_size_per_partition: int, - output_partition_sizes: list[int], - input_size: int, - output_size: int, - params_dtype: torch.dtype, - **extra_weight_attrs, - ): - """Create weights for embedding layer.""" - - weight = Parameter( - torch.empty( - sum(output_partition_sizes), - input_size_per_partition, - dtype=params_dtype, - ), - requires_grad=False, - ) - set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0}) - layer.register_parameter("weight", weight) - set_weight_attrs(weight, extra_weight_attrs) + create_weights = UnquantizedLinearMethod.create_weights def apply( self, layer: torch.nn.Module, x: torch.Tensor, bias: torch.Tensor | None = None @@ -344,71 +322,6 @@ class VocabParallelEmbedding(torch.nn.Module): added_vocab_end_index, ) - def get_sharded_to_full_mapping(self) -> list[int] | None: - """Get a mapping that can be used to reindex the gathered - logits for sampling. - - During sampling, we gather logits from all ranks. The relationship - of index->token_id will follow the same format as outlined in the class - docstring. However, after the gather, we want to reindex the final - logits tensor to map index->token_id one-to-one (the index is always - equal the token_id it corresponds to). The indices returned by this - method allow us to do that. - """ - if self.tp_size < 2: - return None - - base_embeddings: list[int] = [] - added_embeddings: list[int] = [] - padding: list[int] = [] - for tp_rank in range(self.tp_size): - shard_indices = self._get_indices( - self.num_embeddings_padded, - self.org_vocab_size_padded, - self.num_embeddings, - self.org_vocab_size, - tp_rank, - self.tp_size, - ) - range_start = self.num_embeddings_per_partition * tp_rank - range_end = self.num_embeddings_per_partition * (tp_rank + 1) - base_embeddings.extend( - range(range_start, range_start + shard_indices.num_org_elements) - ) - padding.extend( - range( - range_start + shard_indices.num_org_elements, - range_start + shard_indices.num_org_elements_padded, - ) - ) - added_embeddings.extend( - range( - range_start + shard_indices.num_org_elements_padded, - range_start - + shard_indices.num_org_elements_padded - + shard_indices.num_added_elements, - ) - ) - padding.extend( - range( - range_start - + shard_indices.num_org_elements_padded - + shard_indices.num_added_elements, - range_start - + shard_indices.num_org_elements_padded - + shard_indices.num_added_elements_padded, - ) - ) - assert ( - range_start - + shard_indices.num_org_elements_padded - + shard_indices.num_added_elements_padded - == range_end - ) - ret = base_embeddings + added_embeddings + padding - assert len(ret) == self.num_embeddings_padded - return ret - def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor): output_dim = getattr(param, "output_dim", None) packed_dim = getattr(param, "packed_dim", None) diff --git a/python/sglang/multimodal_gen/runtime/loader/component_loaders/text_encoder_loader.py b/python/sglang/multimodal_gen/runtime/loader/component_loaders/text_encoder_loader.py index 0ef27fb542..fd6147e402 100644 --- a/python/sglang/multimodal_gen/runtime/loader/component_loaders/text_encoder_loader.py +++ b/python/sglang/multimodal_gen/runtime/loader/component_loaders/text_encoder_loader.py @@ -5,7 +5,7 @@ from collections.abc import Generator import torch import transformers from torch import nn -from transformers import PretrainedConfig +from transformers import AutoConfig, PretrainedConfig from transformers.utils import SAFE_WEIGHTS_INDEX_NAME from sglang.multimodal_gen.configs.models import EncoderConfig @@ -83,7 +83,6 @@ from sglang.multimodal_gen.runtime.models.registry import ModelRegistry from sglang.multimodal_gen.runtime.platforms import current_platform from sglang.multimodal_gen.runtime.server_args import ServerArgs from sglang.multimodal_gen.runtime.utils.hf_diffusers_utils import ( - get_config, get_diffusers_component_config, load_dict, ) @@ -568,7 +567,7 @@ class TextEncoderLoader(OnlineQuantizationComponentLoader): server_args: ServerArgs, component_name: str, ) -> EncoderConfig: - diffusers_pretrained_config = get_config( + diffusers_pretrained_config = AutoConfig.from_pretrained( component_model_path, trust_remote_code=True ) encoder_index = self._extract_encoder_index( diff --git a/python/sglang/multimodal_gen/runtime/loader/component_loaders/vae_loader.py b/python/sglang/multimodal_gen/runtime/loader/component_loaders/vae_loader.py index 892248b7dc..620e84a3c6 100644 --- a/python/sglang/multimodal_gen/runtime/loader/component_loaders/vae_loader.py +++ b/python/sglang/multimodal_gen/runtime/loader/component_loaders/vae_loader.py @@ -118,8 +118,6 @@ def _convert_conv3d_weights_to_channels_last_3d(module: nn.Module) -> int: Convert Conv3d weights to channels_last_3d (NDHWC) memory format. Returns the number of Conv3d modules converted. """ - if not hasattr(torch, "channels_last_3d"): - return 0 num_converted = 0 for m in module.modules(): if isinstance(m, nn.Conv3d): diff --git a/python/sglang/multimodal_gen/runtime/loader/weight_utils.py b/python/sglang/multimodal_gen/runtime/loader/weight_utils.py index 831d6e0347..6ae0d4f0a7 100644 --- a/python/sglang/multimodal_gen/runtime/loader/weight_utils.py +++ b/python/sglang/multimodal_gen/runtime/loader/weight_utils.py @@ -5,7 +5,6 @@ """Utilities for downloading, loading, initializing and verifying model weights.""" import hashlib -import json import os import tempfile from collections import defaultdict @@ -139,38 +138,6 @@ def get_lock(model_name_or_path: str | Path, cache_dir: str | None = None): return lock -# For models like Mistral-7B-v0.3, there are both sharded -# safetensors files and a consolidated safetensors file. -# Passing both of these to the weight loader functionality breaks. -# So, we use the index_file to -# look up which safetensors files should be used. -def filter_duplicate_safetensors_files( - hf_weights_files: list[str], - hf_folder: str, - index_file: str, - key_filter: Callable[[str], bool] | None = None, -) -> list[str]: - # model.safetensors.index.json is a mapping from keys in the - # torch state_dict to safetensors file holding that weight. - index_file_name = os.path.join(hf_folder, index_file) - if not os.path.isfile(index_file_name): - return hf_weights_files - - # Iterate through the weight_map (weight_name: safetensors files) - # to identify weights that we should use. - with open(index_file_name) as f: - weight_map = json.load(f)["weight_map"] - weight_files_in_index = set() - for weight_name in weight_map: - # remove only shards whose indexed tensors are all filtered - if key_filter is not None and not key_filter(weight_name): - continue - weight_files_in_index.add(os.path.join(hf_folder, weight_map[weight_name])) - # Filter out any fields that are not found in the index file. - hf_weights_files = [f for f in hf_weights_files if f in weight_files_in_index] - return hf_weights_files - - def filter_files_not_needed_for_inference(hf_weights_files: list[str]) -> list[str]: """ Exclude files that are not needed for inference. @@ -376,23 +343,18 @@ def pt_weights_iterator( def default_weight_loader(param: torch.Tensor, loaded_weight: torch.Tensor) -> None: """Default weight loader.""" - try: - if param.numel() == 1 and loaded_weight.numel() == 1: - # Sometimes scalar values aren't considered tensors with shapes - # so if both param and loaded_weight are a scalar, - # "broadcast" instead of copy - param.data.fill_(loaded_weight.item()) - else: - assert param.size() == loaded_weight.size(), ( - f"Attempted to load weight ({loaded_weight.size()}) " - f"into parameter ({param.size()})" - ) + if param.numel() == 1 and loaded_weight.numel() == 1: + # Sometimes scalar values aren't considered tensors with shapes + # so if both param and loaded_weight are a scalar, + # "broadcast" instead of copy + param.data.fill_(loaded_weight.item()) + else: + assert param.size() == loaded_weight.size(), ( + f"Attempted to load weight ({loaded_weight.size()}) " + f"into parameter ({param.size()})" + ) - param.data.copy_(loaded_weight) - except Exception: - # NOTE: This exception is added for the purpose of setting breakpoint to - # debug weight loading issues. - raise + param.data.copy_(loaded_weight) def load_stacked_weight( @@ -485,6 +447,36 @@ def maybe_remap_kv_scale_name(name: str, params_dict: dict) -> str | None: return name +def load_llm_encoder_weights( + weights: Iterable[tuple[str, torch.Tensor]], + params_dict: dict[str, torch.nn.Parameter], + stacked_params_mapping: Iterable[tuple[str, str, str | int]], + *, + strip_prefix: str = "", +) -> set[str]: + """Load Llama/Qwen-style encoder weights, excluding rotary cache tensors.""" + loaded_params: set[str] = set() + for name, loaded_weight in weights: + if strip_prefix and name.startswith(strip_prefix): + name = name[len(strip_prefix) :] + if ( + "rotary_emb.inv_freq" in name + or "rotary_emb.cos_cached" in name + or "rotary_emb.sin_cached" in name + ): + continue + if "scale" in name: + name = maybe_remap_kv_scale_name(name, params_dict) + if name is None: + continue + name = load_stacked_weight( + name, loaded_weight, params_dict, stacked_params_mapping + ) + if name is not None: + loaded_params.add(name) + return loaded_params + + def compute_weights_checksum( named_params: Iterable[tuple[str, torch.Tensor]], ) -> str: diff --git a/python/sglang/multimodal_gen/runtime/managers/forward_context.py b/python/sglang/multimodal_gen/runtime/managers/forward_context.py index f59e1524c3..4fbfd626ae 100644 --- a/python/sglang/multimodal_gen/runtime/managers/forward_context.py +++ b/python/sglang/multimodal_gen/runtime/managers/forward_context.py @@ -2,51 +2,20 @@ # SPDX-License-Identifier: Apache-2.0 # Adapted from vllm: https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/forward_context.py -import time -from collections import defaultdict from contextlib import contextmanager from dataclasses import dataclass -from typing import TYPE_CHECKING, Optional, Type - -import torch - -from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +from typing import TYPE_CHECKING, Optional if TYPE_CHECKING: from sglang.multimodal_gen.runtime.layers.attention import AttentionMetadata from sglang.multimodal_gen.runtime.pipelines_core import Req -logger = init_logger(__name__) - -# TODO(will): check if this is needed -# track_batchsize: bool = envs.SGLANG_DIFFUSION_LOG_BATCHSIZE_INTERVAL >= 0 -track_batchsize: bool = False -last_logging_time: float = 0 -forward_start_time: float = 0 -# batchsize_logging_interval: float = envs.SGLANG_DIFFUSION_LOG_BATCHSIZE_INTERVAL -batchsize_logging_interval: float = 1000 -batchsize_forward_time: defaultdict = defaultdict(list) - @dataclass class ForwardContext: current_timestep: int - # TODO(will): check this arg - # copy from vllm_config.compilation_config.static_forward_context - # attn_layers: Dict[str, Any] - # TODO: extend to support per-layer dynamic forward context attn_metadata: "AttentionMetadata" # set dynamically for each forward pass forward_batch: Optional["Req"] = None - attention_backend_cls: Optional[Type] = None - - def set_attn_backend_cls(self, attention_backend_cls: Type): - if self.attention_backend_cls: - if self.attention_backend_cls != attention_backend_cls: - raise RuntimeError( - f"Different types of attention backend in a same context detected, previous: {self.attention_backend_cls}, new: {attention_backend_cls}" - ) - else: - self.attention_backend_cls = attention_backend_cls _forward_context: Optional["ForwardContext"] = None @@ -65,7 +34,6 @@ def get_forward_context_or_none() -> "ForwardContext | None": return _forward_context -# TODO(will): finalize the interface @contextmanager def set_forward_context( current_timestep, attn_metadata, forward_batch: Optional["Req"] = None @@ -74,10 +42,6 @@ def set_forward_context( can be attention metadata, etc. Here we can inject common logic for every model forward pass. """ - global forward_start_time - need_to_track_batchsize = track_batchsize and attn_metadata is not None - if need_to_track_batchsize: - forward_start_time = time.perf_counter() global _forward_context prev_context = _forward_context _forward_context = ForwardContext( @@ -89,36 +53,4 @@ def set_forward_context( try: yield finally: - global last_logging_time, batchsize_logging_interval - if need_to_track_batchsize: - if hasattr(attn_metadata, "num_prefill_tokens"): - # for v0 attention backends - batchsize = ( - attn_metadata.num_prefill_tokens + attn_metadata.num_decode_tokens - ) - else: - # for v1 attention backends - batchsize = attn_metadata.num_input_tokens - now = time.perf_counter() - # time measurement is in milliseconds - batchsize_forward_time[batchsize].append((now - forward_start_time) * 1000) - if now - last_logging_time > batchsize_logging_interval: - last_logging_time = now - forward_stats = [] - for bs, times in batchsize_forward_time.items(): - if len(times) <= 1: - # can be cudagraph / profiling run - continue - medium = torch.quantile(torch.tensor(times), q=0.5).item() - medium = round(medium, 2) - forward_stats.append((bs, len(times), medium)) - forward_stats.sort(key=lambda x: x[1], reverse=True) - if forward_stats: - logger.info( - ( - "Batchsize forward time stats " - "(batchsize, count, median_time(ms)): %s" - ), - forward_stats, - ) _forward_context = prev_context diff --git a/python/sglang/multimodal_gen/runtime/managers/memory_managers/auto_residency.py b/python/sglang/multimodal_gen/runtime/managers/memory_managers/auto_residency.py index 935904eee5..22f3700bfa 100644 --- a/python/sglang/multimodal_gen/runtime/managers/memory_managers/auto_residency.py +++ b/python/sglang/multimodal_gen/runtime/managers/memory_managers/auto_residency.py @@ -1,25 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 -"""Warmup-calibrated automatic component residency placement. +"""Warmup memory, timing and layer-use estimates for the default workload. -Under ``--performance-mode auto`` with server warmup, each rank measures the -peak GPU memory of bounded synthetic warmup requests and a low-step probe at -the complete default serving shape, then selects a complete serving placement -for every eligible component under the measured memory constraints. - -When no full-shape measurement is available, the fallback estimate splits the -measured peak into persistent weights and workload-scaled activations. Scaling -the whole peak would multiply resident weights by the video frame/area cap -ratio (~16x for Wan-class defaults) and residency adjustment would never trigger. - -The planner targets the model default workload only (default resolution, -default frames, batch=1). Larger shapes, batches, or multi-image inputs need -explicit ``--component-residency``. - -Loading and serving are deliberately separate placement states. The existing -auto policy provides the initial state; when that state can complete loading -and calibration, this module optimizes the long-lived serving state and -validates the transition with a post-placement warmup. It does not force a -single placement to serve two different lifecycle objectives. +Peak-memory extrapolation separates persistent weights from workload-scaled +activations. Phase and layer-use estimates account for stage-specific iteration +counts rather than scaling every stage by the request's denoising steps. """ from __future__ import annotations @@ -29,61 +13,15 @@ from typing import Iterable, Mapping import msgspec -from sglang.multimodal_gen.runtime.managers.memory_managers.component_residency import ( - COMPONENT_OFFLOAD, - LAYERWISE_OFFLOAD, - RESIDENT, -) from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload_components import ( is_dit_component_name, ) -from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger - -logger = init_logger(__name__) GIB_BYTES = 1024**3 # Activation memory rarely scales perfectly linearly with workload units; # pad the extrapolated activation part before checking the budget. ACTIVATION_EXTRAPOLATION_MARGIN = 1.2 -# A target-shape measurement plus the mandatory post-placement warmup justifies -# a tighter reserve than an extrapolated estimate. Both retain an absolute -# floor for allocator slack, shape variance, and CUDA graph or compile pools. -MEASURED_VRAM_RESERVE_FRACTION = 0.05 -EXTRAPOLATED_VRAM_RESERVE_FRACTION = 0.10 -MIN_VRAM_RESERVE_BYTES = 4 * GIB_BYTES -# The absolute floor is sized for datacenter cards, where either fraction can -# dominate. On a 12 GiB card a flat 4 GiB would fence off a third of the device, -# so cap the floor as a share of what is actually there. -MAX_VRAM_RESERVE_FRACTION = 0.20 - -# A feasible placement is not automatically useful. Predictions inside this -# interval are treated as latency-equivalent. The joint optimizer then avoids -# changing strategy, minimizes device memory, preserves the faster estimate, -# and finally minimizes HostPin. The raw estimate is already an upper bound: -# transfer time is capped by the measured request. -ESTIMATED_PINNED_H2D_BYTES_PER_SECOND = 24 * GIB_BYTES -MIN_LATENCY_EQUIVALENCE_NS = 50_000_000 -MAX_LATENCY_EQUIVALENCE_NS = 100_000_000 -LATENCY_EQUIVALENCE_FRACTION = 0.01 -# The transfer model ranks feasible placements; the mandatory warmup is the -# authority on whether a selected placement actually helped. Allow normal -# measurement noise, but undo a round whose calibrated request is materially -# slower than the original layout. -POST_ADJUSTMENT_REGRESSION_FRACTION = 0.05 - -PLACEMENT_STATUS_SKIPPED = "skipped" -PLACEMENT_STATUS_ADJUSTED = "adjusted" -PLACEMENT_STATUS_VALIDATED = "validated" -PLACEMENT_STATUS_ROLLED_BACK = "rolled_back" -PLACEMENT_STATUS_ROLLBACK_FAILED = "rollback_failed" - - -def describe_error(error: BaseException) -> str: - """Never-empty error text (str(AssertionError()) is "" and would be - dropped by any truthiness filter).""" - text = str(error) - return f"{type(error).__name__}: {text}" if text else type(error).__name__ class WarmupMemoryRecord(msgspec.Struct, frozen=True): @@ -113,96 +51,6 @@ class WarmupMemoryRecord(msgspec.Struct, frozen=True): return max(1, self.width) * max(1, self.height) * max(1, self.num_frames) -class ResidencyTarget(msgspec.Struct, frozen=True): - """One complete target state for an auto-managed component.""" - - component_name: str - residency_mode: str - target_resident_weight_bytes: int - # Estimated per-request host-to-device traffic this target removes. - h2d_bytes_per_request: int - # Layerwise candidates jointly choose stage-scoped GPU residency and host - # pinning. None is used by ordinary component placement. - target_layerwise_resident_layers: tuple[int, ...] | None = None - target_layerwise_pinned_layers: tuple[tuple[int, ...], ...] | None = None - pinned_host_delta_bytes: int = 0 - host_unpin_scratch_bytes: int = 0 - host_pin_scratch_bytes: int = 0 - host_materialize_scratch_bytes: int = 0 - # Signed device-memory delta while applying the placement before the - # validation warmup. Layerwise -> resident materializes every managed - # layer immediately; a demotion can release those bytes first and fund a - # later materialization in the same transaction. - device_transition_delta_bytes: int = 0 - permanent_residency: bool = False - # Device-memory delta relative to the measured placement. A component - # already loaded for its own phase has a different delta from phases where - # it is absent; keeping both avoids adding the same weights twice. - active_device_delta_bytes: int = 0 - # Delta when the component is already present because of async prefetch, - # but is not the semantic owner of this phase. - present_device_delta_bytes: int = 0 - inactive_device_delta_bytes: int = 0 - # None preserves the historical derived target for hand-built callers: - # partial layerwise targets remain layerwise, every other option is - # resident. Generated complete-state frontiers set this explicitly. - target_residency_mode: str | None = None - current_placement: bool = False - target_device_weight_bytes: int = 0 - target_pinned_host_bytes: int = 0 - - def target_mode(self) -> str: - if self.target_residency_mode is not None: - return self.target_residency_mode - if ( - self.target_layerwise_resident_layers is not None - and not self.permanent_residency - ): - return LAYERWISE_OFFLOAD - return RESIDENT - - def option_key(self) -> str: - target_mode = self.target_mode() - if target_mode == COMPONENT_OFFLOAD: - return f"{self.component_name}:component-offload" - if self.target_layerwise_resident_layers is None: - return f"{self.component_name}:resident" - layer_counts = ",".join( - str(count) for count in self.target_layerwise_resident_layers - ) - pinned = "|".join( - ",".join(str(index) for index in indices) or "-" - for indices in self.target_layerwise_pinned_layers or () - ) - permanence = "permanent" if self.permanent_residency else "stage" - return f"{self.component_name}:{permanence}:layers={layer_counts}:pins={pinned}" - - -class RankResidencyReport(msgspec.Struct, frozen=True): - """One rank's inputs to the replica-wide placement decision.""" - - rank: int - budget_bytes: int - estimated_peak_bytes: int | None - target_workload_measured: bool = False - observed_reserved_bytes: int = 0 - estimated_peak_bytes_by_phase: dict[str, int] = {} - active_components_by_phase: dict[str, tuple[str, ...]] = {} - used_components_by_phase: dict[str, tuple[str, ...]] = {} - full_weight_transition_components_by_phase: dict[str, tuple[str, ...]] = {} - current_device_weight_bytes_by_component: dict[str, int] = {} - node_rank: int = 0 - pinned_host_bytes: int = 0 - host_pin_capacity_bytes: int = 0 - host_transition_headroom_bytes: int = 0 - device_transition_allocated_bytes: int = 0 - estimated_request_duration_ns: int = 0 - measured_request_duration_ns: int = 0 - candidate_latency_savings_ns: dict[str, int] = {} - candidates: list[ResidencyTarget] = [] - skip_reason: str | None = None - - def estimate_layerwise_layer_uses( *, records: Iterable[WarmupMemoryRecord], diff --git a/python/sglang/multimodal_gen/runtime/managers/memory_managers/component_loading_order.py b/python/sglang/multimodal_gen/runtime/managers/memory_managers/component_loading_order.py index 890eaf92e7..44481df9fd 100644 --- a/python/sglang/multimodal_gen/runtime/managers/memory_managers/component_loading_order.py +++ b/python/sglang/multimodal_gen/runtime/managers/memory_managers/component_loading_order.py @@ -12,6 +12,7 @@ import os from dataclasses import dataclass from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload_components import ( + component_base_name, is_dit_component_name, is_image_encoder_component_name, is_text_encoder_component_name, @@ -34,13 +35,6 @@ class ComponentLoadSpec: _WEIGHT_FILE_SUFFIXES = (".bin", ".pt", ".pth") -def _component_base_name(component_name: str) -> str: - prefix, separator, suffix = component_name.rpartition("_") - if separator and suffix.isdigit(): - return prefix - return component_name - - def _component_variant_priority(component_name: str) -> int: _, separator, suffix = component_name.rpartition("_") if separator and suffix.isdigit(): @@ -50,7 +44,7 @@ def _component_variant_priority(component_name: str) -> int: def component_load_risk_rank(component_name: str) -> int: """Fallback type rank when checkpoint size cannot be inferred.""" - candidate_names = (component_name, _component_base_name(component_name)) + candidate_names = (component_name, component_base_name(component_name)) if any(is_dit_component_name(name) for name in candidate_names): return 0 if any(is_text_encoder_component_name(name) for name in candidate_names): diff --git a/python/sglang/multimodal_gen/runtime/models/dits/causal_wanvideo.py b/python/sglang/multimodal_gen/runtime/models/dits/causal_wanvideo.py index 7eb8e9b8f4..8e58f39c3d 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/causal_wanvideo.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/causal_wanvideo.py @@ -98,7 +98,6 @@ class CausalWanSelfAttention(nn.Module): self.sink_size = sink_size self.qk_norm = qk_norm self.eps = eps - self.parallel_attention = parallel_attention # Scaled dot product attention self.attn = LocalAttention( diff --git a/python/sglang/multimodal_gen/runtime/models/dits/cosmos3video.py b/python/sglang/multimodal_gen/runtime/models/dits/cosmos3video.py index bcd45d75c3..183edf0637 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/cosmos3video.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/cosmos3video.py @@ -375,7 +375,6 @@ class DomainAwareLinear(nn.Module): super().__init__() self.input_size = input_size self.output_size = output_size - self.num_domains = num_domains self.fc = nn.Embedding(num_domains, output_size * input_size) self.bias = nn.Embedding(num_domains, output_size) nn.init.xavier_uniform_( @@ -583,6 +582,27 @@ def _build_mlp( # ----------------------------------------------------------------------------- +def _initialize_attention_projections(module, prefix, quant_config): + module.q_size = module.num_attention_heads * module.head_dim + module.kv_size = module.num_key_value_heads * module.head_dim + module.to_qkv = MergedColumnParallelLinear( + module.hidden_size, + [module.q_size, module.kv_size, module.kv_size], + bias=False, + gather_output=False, + quant_config=quant_config, + prefix=add_prefix("to_qkv", prefix), + ) + module.to_out = RowParallelLinear( + module.q_size, + module.hidden_size, + bias=False, + input_is_parallel=True, + quant_config=quant_config, + prefix=add_prefix("to_out", prefix), + ) + + class Cosmos3CausalAttention(nn.Module): """Understanding pathway: causal self-attention on text tokens.""" @@ -618,24 +638,7 @@ class Cosmos3CausalAttention(nn.Module): self.local_num_attention_heads = num_attention_heads // self.tp_size self.local_num_key_value_heads = num_key_value_heads // self.tp_size - self.q_size = num_attention_heads * head_dim - self.kv_size = num_key_value_heads * head_dim - self.to_qkv = MergedColumnParallelLinear( - hidden_size, - [self.q_size, self.kv_size, self.kv_size], - bias=False, - gather_output=False, - quant_config=quant_config, - prefix=add_prefix("to_qkv", prefix), - ) - self.to_out = RowParallelLinear( - num_attention_heads * head_dim, - hidden_size, - bias=False, - input_is_parallel=True, - quant_config=quant_config, - prefix=add_prefix("to_out", prefix), - ) + _initialize_attention_projections(self, prefix, quant_config) # Per-head QK norm (optional; some backbones omit it on text). if qk_norm: @@ -754,24 +757,7 @@ class Cosmos3CrossAttention(nn.Module): self.local_num_attention_heads = num_attention_heads // self.tp_size self.local_num_key_value_heads = num_key_value_heads // self.tp_size - self.q_size = num_attention_heads * head_dim - self.kv_size = num_key_value_heads * head_dim - self.to_qkv = MergedColumnParallelLinear( - hidden_size, - [self.q_size, self.kv_size, self.kv_size], - bias=False, - gather_output=False, - quant_config=quant_config, - prefix=add_prefix("to_qkv", prefix), - ) - self.to_out = RowParallelLinear( - num_attention_heads * head_dim, - hidden_size, - bias=False, - input_is_parallel=True, - quant_config=quant_config, - prefix=add_prefix("to_out", prefix), - ) + _initialize_attention_projections(self, prefix, quant_config) self.norm_q = RMSNorm(head_dim, eps=1e-6) self.norm_k = RMSNorm(head_dim, eps=1e-6) diff --git a/python/sglang/multimodal_gen/runtime/models/dits/flux.py b/python/sglang/multimodal_gen/runtime/models/dits/flux.py index a5c2e9d353..584960c576 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/flux.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/flux.py @@ -523,7 +523,6 @@ class FluxAttention(torch.nn.Module, AttentionModuleMixin): ) self.local_heads = divide(self.heads, self.tp_size) self.added_kv_proj_dim = added_kv_proj_dim - self.added_proj_bias = added_proj_bias self.use_fused_qkv = isinstance(quant_config, NunchakuConfig) self.use_fused_added_qkv = isinstance(quant_config, NunchakuConfig) diff --git a/python/sglang/multimodal_gen/runtime/models/dits/flux_2.py b/python/sglang/multimodal_gen/runtime/models/dits/flux_2.py index f839d78bef..3b24523df8 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/flux_2.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/flux_2.py @@ -573,7 +573,6 @@ class Flux2Attention(torch.nn.Module, AttentionModuleMixin): self.dropout = dropout self.added_kv_proj_dim = added_kv_proj_dim - self.added_proj_bias = added_proj_bias # Packed NVFP4 checkpoints already serialize QKV together. ModelOpt # FP8 exports separate Diffusers tensors, but the loader can merge diff --git a/python/sglang/multimodal_gen/runtime/models/dits/hunyuanvideo.py b/python/sglang/multimodal_gen/runtime/models/dits/hunyuanvideo.py index 9e064ccbf1..04288d6412 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/hunyuanvideo.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/hunyuanvideo.py @@ -669,7 +669,6 @@ class HunyuanVideoTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixi self.out_channels = ( config.in_channels if config.out_channels is None else config.out_channels ) - self.unpatchify_channels = self.out_channels self.guidance_embeds = config.guidance_embeds self.rope_dim_list = list(config.rope_axes_dim) self.rope_theta = config.rope_theta diff --git a/python/sglang/multimodal_gen/runtime/models/dits/lingbot_world.py b/python/sglang/multimodal_gen/runtime/models/dits/lingbot_world.py index 18eca688d6..2ec1730311 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/lingbot_world.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/lingbot_world.py @@ -704,16 +704,9 @@ class LingBotWorldTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixi width_local: int, device: torch.device, ) -> tuple[torch.Tensor, torch.Tensor]: - token_start = rank * local_len - token_indices = torch.arange( - token_start, token_start + local_len, device=device, dtype=torch.long + return self.rotary_emb.forward_3d_sequence_shard( + local_len, rank, frame_stride_local, width_local, device ) - t_idx = token_indices // frame_stride_local - rem = token_indices % frame_stride_local - h_idx = rem // width_local - w_idx = rem % width_local - positions = torch.stack((t_idx, h_idx, w_idx), dim=1) - return self.rotary_emb.forward_uncached(positions) def forward( self, diff --git a/python/sglang/multimodal_gen/runtime/models/dits/ltx_2.py b/python/sglang/multimodal_gen/runtime/models/dits/ltx_2.py index 18ab9df5f7..41715ba9ae 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/ltx_2.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/ltx_2.py @@ -520,11 +520,6 @@ class LTX2AudioVideoRotaryPosEmbed(nn.Module): audio_coords = audio_coords.unsqueeze(1) return audio_coords - def prepare_coords(self, *args, **kwargs): - if self.modality == "video": - return self.prepare_video_coords(*args, **kwargs) - return self.prepare_audio_coords(*args, **kwargs) - def forward( self, coords: torch.Tensor, @@ -697,7 +692,6 @@ class LTX2TPRMSNormAcrossHeads(nn.Module): ) -> None: super().__init__() self.full_hidden_size = full_hidden_size - self.local_hidden_size = local_hidden_size self.eps = eps self.weight = nn.Parameter(torch.ones(local_hidden_size)) @@ -1864,8 +1858,6 @@ class LTX2VideoTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixin): num_attention_heads=self.audio_num_attention_heads, ) - self.cross_pe_max_pos = cross_attn_pos_embed_max_pos - # 5. Transformer Blocks self.transformer_blocks = nn.ModuleList( [ diff --git a/python/sglang/multimodal_gen/runtime/models/dits/mova_audio_dit.py b/python/sglang/multimodal_gen/runtime/models/dits/mova_audio_dit.py index 0dc76d999f..1983b53001 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/mova_audio_dit.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/mova_audio_dit.py @@ -15,7 +15,6 @@ from torch.distributed.tensor import DTensor from sglang.multimodal_gen.configs.models.dits.mova_audio import MOVAAudioConfig from sglang.multimodal_gen.configs.models.fsdp import is_block from sglang.multimodal_gen.runtime.layers.linear import ReplicatedLinear -from sglang.multimodal_gen.runtime.layers.mlp import MLP from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config import ( QuantizationConfig, ) @@ -25,7 +24,11 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload im from sglang.multimodal_gen.runtime.models.dits.base import CachableDiT # Reuse common functions and classes from mova_video_dit -from .mova_video_dit import DiTBlock, precompute_freqs_cis, sinusoidal_embedding_1d +from .mova_video_dit import ( + initialize_mova_transformer_layers, + precompute_freqs_cis, + sinusoidal_embedding_1d, +) def precompute_freqs_cis_1d(dim: int, end: int = 16384, theta: float = 10000.0): @@ -102,15 +105,11 @@ class WanAudioModel(CachableDiT, LayerwiseOffloadableModuleMixin): # Extract parameters from config dim = config.dim in_dim = config.in_dim - ffn_dim = config.ffn_dim out_dim = config.out_dim - text_dim = config.text_dim freq_dim = config.freq_dim eps = config.eps patch_size = config.patch_size num_heads = config.num_heads - num_layers = config.num_layers - has_image_pos_emb = config.has_image_pos_emb has_ref_conv = config.has_ref_conv separated_timestep = config.separated_timestep require_vae_embedding = config.require_vae_embedding @@ -126,38 +125,15 @@ class WanAudioModel(CachableDiT, LayerwiseOffloadableModuleMixin): self.require_clip_embedding = require_clip_embedding self.fuse_vae_embedding_in_latents = fuse_vae_embedding_in_latents self.vae_type = vae_type - # self.patch_embedding = nn.Conv3d( - # in_dim, dim, kernel_size=patch_size, stride=patch_size) self.patch_embedding = Conv1dLocalIsland( in_dim, dim, kernel_size=patch_size, stride=patch_size ) - self.text_embedding = MLP( - text_dim, - dim, - output_dim=dim, - act_type="gelu_pytorch_tanh", - quant_config=quant_config, - ) - self.time_embedding = MLP( - freq_dim, dim, output_dim=dim, act_type="silu", quant_config=quant_config - ) - # Preserve state_dict keys (time_projection.1.weight/bias). - self.time_projection = nn.Sequential( - nn.SiLU(), ReplicatedLinear(dim, dim * 6, quant_config=quant_config) - ) - self.blocks = nn.ModuleList( - [ - DiTBlock(dim, num_heads, ffn_dim, eps, quant_config=quant_config) - for _ in range(num_layers) - ] - ) + initialize_mova_transformer_layers(self, config, quant_config) self.head = Head(dim, out_dim, patch_size, eps) self.num_heads = num_heads self.freqs = None - self.img_pos_emb = None if has_ref_conv: self.ref_conv = nn.Conv2d(16, dim, kernel_size=(2, 2), stride=(2, 2)) - self.has_image_pos_emb = has_image_pos_emb self.has_ref_conv = has_ref_conv self.hidden_size = dim self.num_attention_heads = num_heads diff --git a/python/sglang/multimodal_gen/runtime/models/dits/mova_video_dit.py b/python/sglang/multimodal_gen/runtime/models/dits/mova_video_dit.py index eff8494627..9f8572d3dc 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/mova_video_dit.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/mova_video_dit.py @@ -272,9 +272,6 @@ class CrossAttention(_MOVAAttention): class MulAdd(nn.Module): - def __init__(self): - super().__init__() - def forward(self, x, gate, residual): return residual + gate * x @@ -408,6 +405,41 @@ class Conv3dLocalIsland(nn.Conv3d): return super().forward(input) +def initialize_mova_transformer_layers(model, config, quant_config): + """Create shared audio/video layers in checkpoint registration order.""" + model.text_embedding = MLP( + config.text_dim, + config.dim, + output_dim=config.dim, + act_type="gelu_pytorch_tanh", + quant_config=quant_config, + ) + model.time_embedding = MLP( + config.freq_dim, + config.dim, + output_dim=config.dim, + act_type="silu", + quant_config=quant_config, + ) + # Preserve state_dict keys (time_projection.1.weight/bias). + model.time_projection = nn.Sequential( + nn.SiLU(), + ReplicatedLinear(config.dim, config.dim * 6, quant_config=quant_config), + ) + model.blocks = nn.ModuleList( + [ + DiTBlock( + config.dim, + config.num_heads, + config.ffn_dim, + config.eps, + quant_config=quant_config, + ) + for _ in range(config.num_layers) + ] + ) + + class WanModel(CachableDiT, LayerwiseOffloadableModuleMixin): _fsdp_shard_conditions = [is_block] _compile_conditions = [is_block] @@ -426,15 +458,11 @@ class WanModel(CachableDiT, LayerwiseOffloadableModuleMixin): # Extract parameters from config dim = config.dim in_dim = config.in_dim - ffn_dim = config.ffn_dim out_dim = config.out_dim - text_dim = config.text_dim freq_dim = config.freq_dim eps = config.eps patch_size = config.patch_size num_heads = config.num_heads - num_layers = config.num_layers - has_image_pos_emb = config.has_image_pos_emb has_ref_conv = config.has_ref_conv separated_timestep = config.separated_timestep require_vae_embedding = config.require_vae_embedding @@ -452,33 +480,13 @@ class WanModel(CachableDiT, LayerwiseOffloadableModuleMixin): self.patch_embedding = Conv3dLocalIsland( in_dim, dim, kernel_size=patch_size, stride=patch_size ) - self.text_embedding = MLP( - text_dim, - dim, - output_dim=dim, - act_type="gelu_pytorch_tanh", - quant_config=quant_config, - ) - self.time_embedding = MLP( - freq_dim, dim, output_dim=dim, act_type="silu", quant_config=quant_config - ) - # Preserve state_dict keys (time_projection.1.weight/bias). - self.time_projection = nn.Sequential( - nn.SiLU(), ReplicatedLinear(dim, dim * 6, quant_config=quant_config) - ) - self.blocks = nn.ModuleList( - [ - DiTBlock(dim, num_heads, ffn_dim, eps, quant_config=quant_config) - for _ in range(num_layers) - ] - ) + initialize_mova_transformer_layers(self, config, quant_config) self.head = Head(dim, out_dim, patch_size, eps) self.num_heads = num_heads self.freqs = None if has_ref_conv: self.ref_conv = nn.Conv2d(16, dim, kernel_size=(2, 2), stride=(2, 2)) - self.has_image_pos_emb = has_image_pos_emb self.has_ref_conv = has_ref_conv self.hidden_size = dim self.num_attention_heads = num_heads diff --git a/python/sglang/multimodal_gen/runtime/models/dits/qwen_image.py b/python/sglang/multimodal_gen/runtime/models/dits/qwen_image.py index 71c9bd2542..f906c6ab39 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/qwen_image.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/qwen_image.py @@ -744,7 +744,6 @@ class QwenImageCrossAttention(nn.Module): self.window_size = window_size self.qk_norm = qk_norm self.eps = eps - self.parallel_attention = parallel_attention self.added_kv_proj_dim = added_kv_proj_dim self.prefix = prefix self.defer_output_bias = _defer_modelopt_output_bias(quant_config) diff --git a/python/sglang/multimodal_gen/runtime/models/dits/sana_wm.py b/python/sglang/multimodal_gen/runtime/models/dits/sana_wm.py index 2365fe91f1..aac01d8ef0 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/sana_wm.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/sana_wm.py @@ -169,6 +169,17 @@ class SanaWMBlock(nn.Module): tokens_per_frame = N // num_frames return x.reshape(B, num_frames, tokens_per_frame, C), tokens_per_frame + def _prepare_modulation(self, t: torch.Tensor, batch_size: int): + if t.dim() == 2: + return None, ( + self.scale_shift_table[None] + t.reshape(batch_size, 6, -1) + ).chunk(6, dim=1) + num_frames = t.reshape(batch_size, -1, 6, t.shape[-1] // 6).shape[1] + t = t.reshape(batch_size, num_frames, 6, -1) + return num_frames, (self.scale_shift_table[None, None, :, :] + t).chunk( + 6, dim=2 + ) + def forward( self, x: torch.Tensor, # (B, N, D) @@ -184,17 +195,8 @@ class SanaWMBlock(nn.Module): chunk_index: Optional[List[int]] = None, ) -> torch.Tensor: B = x.shape[0] - if t.dim() == 2: - num_frames = None - shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = ( - self.scale_shift_table[None] + t.reshape(B, 6, -1) - ).chunk(6, dim=1) - else: - num_frames = t.reshape(B, -1, 6, t.shape[-1] // 6).shape[1] - t = t.reshape(B, num_frames, 6, -1) - shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = ( - self.scale_shift_table[None, None, :, :] + t - ).chunk(6, dim=2) + num_frames, modulation = self._prepare_modulation(t, B) + shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = modulation # Self-attention with UCPE camera branch if num_frames is None: @@ -259,17 +261,8 @@ class SanaWMBlock(nn.Module): ) -> Tuple[torch.Tensor, list]: """Streaming counterpart of ``forward``: threads the per-block 10-slot ``kv_cache`` through cached attention + FFN.""" B = x.shape[0] - if t.dim() == 2: - num_frames = None - shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = ( - self.scale_shift_table[None] + t.reshape(B, 6, -1) - ).chunk(6, dim=1) - else: - num_frames = t.reshape(B, -1, 6, t.shape[-1] // 6).shape[1] - t = t.reshape(B, num_frames, 6, -1) - shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = ( - self.scale_shift_table[None, None, :, :] + t - ).chunk(6, dim=2) + num_frames, modulation = self._prepare_modulation(t, B) + shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = modulation if num_frames is None: x_in = self._modulate(self.norm1(x), shift_msa, scale_msa) @@ -435,7 +428,6 @@ class SanaWMTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixin): for i in range(depth) if arch.softmax_every_n > 0 and (i + 1) % arch.softmax_every_n == 0 ) - self.softmax_block_indices = tuple(sorted(softmax_idx)) self.blocks = nn.ModuleList( [ diff --git a/python/sglang/multimodal_gen/runtime/models/dits/wanvideo.py b/python/sglang/multimodal_gen/runtime/models/dits/wanvideo.py index ec83bfc922..66e40dc2c8 100755 --- a/python/sglang/multimodal_gen/runtime/models/dits/wanvideo.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/wanvideo.py @@ -203,7 +203,6 @@ class WanSelfAttention(nn.Module): self.window_size = window_size self.qk_norm = qk_norm self.eps = eps - self.parallel_attention = parallel_attention tp_size = get_tp_world_size() # layers @@ -1137,19 +1136,9 @@ class WanTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixin): width_local: int, device: torch.device, ) -> tuple[torch.Tensor, torch.Tensor]: - token_start = rank * local_len - token_indices = torch.arange( - token_start, - token_start + local_len, - device=device, - dtype=torch.long, + return self.rotary_emb.forward_3d_sequence_shard( + local_len, rank, frame_stride_local, width_local, device ) - t_idx = token_indices // frame_stride_local - rem = token_indices % frame_stride_local - h_idx = rem // width_local - w_idx = rem % width_local - positions = torch.stack((t_idx, h_idx, w_idx), dim=1) - return self.rotary_emb.forward_uncached(positions) def forward( self, diff --git a/python/sglang/multimodal_gen/runtime/models/dits/zimage.py b/python/sglang/multimodal_gen/runtime/models/dits/zimage.py index d976e6e141..8226062724 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/zimage.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/zimage.py @@ -1379,10 +1379,6 @@ class ZImageTransformer2DModel(CachableDiT, LayerwiseOffloadableModuleMixin): self._cached_joint_attn_mask_meta = (cache_key, result) return self._pin_for_active_capture(result) - @staticmethod - def _has_padding(valid_lens: list[int], target_len: int) -> bool: - return any(int(length) < target_len for length in valid_lens) - @staticmethod def _as_image_list(hidden_states) -> list[torch.Tensor]: """Normalize 4D/5D image latents into per-sample tensors.""" diff --git a/python/sglang/multimodal_gen/runtime/models/encoders/base.py b/python/sglang/multimodal_gen/runtime/models/encoders/base.py index 21eaae1905..c191c9a6d8 100644 --- a/python/sglang/multimodal_gen/runtime/models/encoders/base.py +++ b/python/sglang/multimodal_gen/runtime/models/encoders/base.py @@ -31,6 +31,18 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload im from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum +def get_attention_head_partition( + num_heads: int, num_kv_heads: int, tp_size: int +) -> tuple[int, int]: + """Partition query heads and shard or replicate KV heads across TP ranks.""" + assert num_heads % tp_size == 0 + if num_kv_heads >= tp_size: + assert num_kv_heads % tp_size == 0 + else: + assert tp_size % num_kv_heads == 0 + return num_heads // tp_size, max(1, num_kv_heads // tp_size) + + def get_folding_tp_group(config: EncoderConfig): """Return the TP group selected for an encoder.""" mode = config.parallel_folding_mode @@ -218,7 +230,6 @@ class TextEncoder( # shard conditions left in the root group stays sharded unless the entry # point is registered; loaders read this and register each name. _fsdp_forward_methods: tuple[str, ...] = () - _stacked_params_mapping: list[tuple[str, str, str]] = field(default_factory=list) _supported_attention_backends: set[AttentionBackendEnum] = ( TextEncoderConfig()._supported_attention_backends ) @@ -227,7 +238,6 @@ class TextEncoder( super().__init__() self.config = config self._fsdp_shard_conditions = config.arch_config._fsdp_shard_conditions - self._stacked_params_mapping = config.arch_config.stacked_params_mapping if not self.supported_attention_backends: raise ValueError( f"Subclass {self.__class__.__name__} must define _supported_attention_backends" diff --git a/python/sglang/multimodal_gen/runtime/models/encoders/gemma2.py b/python/sglang/multimodal_gen/runtime/models/encoders/gemma2.py index 7edcc82a47..0511c9868f 100644 --- a/python/sglang/multimodal_gen/runtime/models/encoders/gemma2.py +++ b/python/sglang/multimodal_gen/runtime/models/encoders/gemma2.py @@ -40,6 +40,7 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload im ) from sglang.multimodal_gen.runtime.models.encoders.base import ( EncoderTensorParallelMixin, + get_attention_head_partition, ) logger = logging.getLogger(__name__) @@ -114,14 +115,10 @@ class Gemma2Attention(nn.Module): self.hidden_size = hidden_size tp_size = get_tp_world_size() self.total_num_heads = num_heads - assert self.total_num_heads % tp_size == 0 - self.num_heads = self.total_num_heads // tp_size self.total_num_kv_heads = num_kv_heads - if self.total_num_kv_heads >= tp_size: - assert self.total_num_kv_heads % tp_size == 0 - else: - assert tp_size % self.total_num_kv_heads == 0 - self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) + self.num_heads, self.num_kv_heads = get_attention_head_partition( + num_heads, num_kv_heads, tp_size + ) arch = config.arch_config self.head_dim = arch.head_dim diff --git a/python/sglang/multimodal_gen/runtime/models/encoders/gemma_3.py b/python/sglang/multimodal_gen/runtime/models/encoders/gemma_3.py index 5dc62b7c69..a4d4c3edf1 100644 --- a/python/sglang/multimodal_gen/runtime/models/encoders/gemma_3.py +++ b/python/sglang/multimodal_gen/runtime/models/encoders/gemma_3.py @@ -27,6 +27,7 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload im ) from sglang.multimodal_gen.runtime.models.encoders.base import ( EncoderTensorParallelMixin, + get_attention_head_partition, ) from sglang.multimodal_gen.runtime.utils.common import add_prefix from sglang.srt.models.siglip import SiglipVisionModel @@ -68,6 +69,36 @@ def get_attention_sliding_window_size(config): return config.sliding_window - 1 +def _load_with_shard_id( + weight_loader, param, loaded_weight: torch.Tensor, shard_id +) -> None: + """Try the supplied shard ID, then its equivalent q/k/v or integer form.""" + try: + weight_loader(param, loaded_weight, shard_id) + return + except (AssertionError, TypeError): + pass + + if isinstance(shard_id, str): + mapping = {"q": 0, "k": 1, "v": 2} + if shard_id in mapping: + weight_loader(param, loaded_weight, mapping[shard_id]) + return + if shard_id.isdigit(): + weight_loader(param, loaded_weight, int(shard_id)) + return + elif isinstance(shard_id, int): + mapping = {0: "q", 1: "k", 2: "v"} + if shard_id in mapping: + weight_loader(param, loaded_weight, mapping[shard_id]) + return + + raise TypeError( + f"Unsupported shard_id={shard_id!r} for weight_loader={weight_loader} " + f"(param={getattr(param, 'name', '')})." + ) + + class Gemma3RMSNorm(nn.Module): def __init__(self, dim: int, eps: float = 1e-6): super().__init__() @@ -146,14 +177,10 @@ class Gemma3Attention(nn.Module): self.hidden_size = hidden_size tp_size = get_tp_world_size() self.total_num_heads = num_heads - assert self.total_num_heads % tp_size == 0 - self.num_heads = self.total_num_heads // tp_size self.total_num_kv_heads = num_kv_heads - if self.total_num_kv_heads >= tp_size: - assert self.total_num_kv_heads % tp_size == 0 - else: - assert tp_size % self.total_num_kv_heads == 0 - self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) + self.num_heads, self.num_kv_heads = get_attention_head_partition( + num_heads, num_kv_heads, tp_size + ) self.head_dim = getattr( config.text_config, "head_dim", self.hidden_size // self.total_num_heads diff --git a/python/sglang/multimodal_gen/runtime/models/encoders/llama.py b/python/sglang/multimodal_gen/runtime/models/encoders/llama.py index 2b8190cb25..76d927fb20 100644 --- a/python/sglang/multimodal_gen/runtime/models/encoders/llama.py +++ b/python/sglang/multimodal_gen/runtime/models/encoders/llama.py @@ -51,10 +51,12 @@ from sglang.multimodal_gen.runtime.layers.vocab_parallel_embedding import ( VocabParallelEmbedding, ) from sglang.multimodal_gen.runtime.loader.weight_utils import ( - load_stacked_weight, - maybe_remap_kv_scale_name, + load_llm_encoder_weights, +) +from sglang.multimodal_gen.runtime.models.encoders.base import ( + TextEncoder, + get_attention_head_partition, ) -from sglang.multimodal_gen.runtime.models.encoders.base import TextEncoder class LlamaMLP(nn.Module): @@ -116,18 +118,10 @@ class LlamaAttention(nn.Module): self.hidden_size = hidden_size tp_size = get_tp_world_size() self.total_num_heads = num_heads - assert self.total_num_heads % tp_size == 0 - self.num_heads = self.total_num_heads // tp_size self.total_num_kv_heads = num_kv_heads - if self.total_num_kv_heads >= tp_size: - # Number of KV heads is greater than TP size, so we partition - # the KV heads across multiple tensor parallel GPUs. - assert self.total_num_kv_heads % tp_size == 0 - else: - # Number of KV heads is less than TP size, so we replicate - # the KV heads across multiple tensor parallel GPUs. - assert tp_size % self.total_num_kv_heads == 0 - self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) + self.num_heads, self.num_kv_heads = get_attention_head_partition( + num_heads, num_kv_heads, tp_size + ) # MistralConfig has an optional head_dim introduced by Mistral-Nemo self.head_dim = getattr( config, "head_dim", self.hidden_size // self.total_num_heads @@ -390,32 +384,11 @@ class LlamaModel(TextEncoder): return output def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - for name, loaded_weight in weights: - if "rotary_emb.inv_freq" in name: - continue - if "rotary_emb.cos_cached" in name or "rotary_emb.sin_cached" in name: - # Models trained using ColossalAI may include these tensors in - # the checkpoint. Skip them. - continue - if "scale" in name: - # Remapping the name of FP8 kv-scale. - kv_scale_name: str | None = maybe_remap_kv_scale_name(name, params_dict) - if kv_scale_name is None: - continue - else: - name = kv_scale_name - name = load_stacked_weight( - name, - loaded_weight, - params_dict, - self.config.arch_config.stacked_params_mapping, - ) - if name is not None: - loaded_params.add(name) - return loaded_params + return load_llm_encoder_weights( + weights, + dict(self.named_parameters()), + self.config.arch_config.stacked_params_mapping, + ) EntryClass = LlamaModel diff --git a/python/sglang/multimodal_gen/runtime/models/encoders/qwen2_5vl.py b/python/sglang/multimodal_gen/runtime/models/encoders/qwen2_5vl.py index 1db47cbfe1..130219d475 100644 --- a/python/sglang/multimodal_gen/runtime/models/encoders/qwen2_5vl.py +++ b/python/sglang/multimodal_gen/runtime/models/encoders/qwen2_5vl.py @@ -1332,13 +1332,10 @@ class Qwen2_5_VLForConditionalGeneration(TextEncoder): break if loaded_stacked_param: continue - try: - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - param = params_dict[name] - except KeyError: - raise + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + param = params_dict[name] weight_loader = getattr(param, "weight_loader", default_weight_loader) loaded_weight = loaded_weight.to(param.dtype) diff --git a/python/sglang/multimodal_gen/runtime/models/encoders/qwen3.py b/python/sglang/multimodal_gen/runtime/models/encoders/qwen3.py index b74d93c7fb..1a9f4d6c05 100644 --- a/python/sglang/multimodal_gen/runtime/models/encoders/qwen3.py +++ b/python/sglang/multimodal_gen/runtime/models/encoders/qwen3.py @@ -20,10 +20,12 @@ from sglang.multimodal_gen.runtime.layers.vocab_parallel_embedding import ( VocabParallelEmbedding, ) from sglang.multimodal_gen.runtime.loader.weight_utils import ( - load_stacked_weight, - maybe_remap_kv_scale_name, + load_llm_encoder_weights, +) +from sglang.multimodal_gen.runtime.models.encoders.base import ( + TextEncoder, + get_attention_head_partition, ) -from sglang.multimodal_gen.runtime.models.encoders.base import TextEncoder from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.layernorm import RMSNorm @@ -91,14 +93,10 @@ class Qwen3Attention(nn.Module): self.hidden_size = hidden_size tp_size = get_tp_world_size() self.total_num_heads = num_heads - assert self.total_num_heads % tp_size == 0 - self.num_heads = self.total_num_heads // tp_size self.total_num_kv_heads = num_kv_heads - if self.total_num_kv_heads >= tp_size: - assert self.total_num_kv_heads % tp_size == 0 - else: - assert tp_size % self.total_num_kv_heads == 0 - self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) + self.num_heads, self.num_kv_heads = get_attention_head_partition( + num_heads, num_kv_heads, tp_size + ) self.head_dim = getattr( config, "head_dim", self.hidden_size // self.total_num_heads @@ -432,38 +430,12 @@ class Qwen3ForCausalLM(TextEncoder): def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: """Load weights with support for tensor parallelism and weight remapping.""" - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - - for name, loaded_weight in weights: - # Strip 'model.' prefix from HuggingFace Qwen3 weights - if name.startswith("model."): - name = name[6:] # len("model.") == 6 - - # Skip rotary embedding weights - if "rotary_emb.inv_freq" in name: - continue - if "rotary_emb.cos_cached" in name or "rotary_emb.sin_cached" in name: - continue - - # Handle KV scale remapping - if "scale" in name: - kv_scale_name: str | None = maybe_remap_kv_scale_name(name, params_dict) - if kv_scale_name is None: - continue - else: - name = kv_scale_name - - name = load_stacked_weight( - name, - loaded_weight, - params_dict, - self.config.arch_config.stacked_params_mapping, - ) - if name is not None: - loaded_params.add(name) - - return loaded_params + return load_llm_encoder_weights( + weights, + dict(self.named_parameters()), + self.config.arch_config.stacked_params_mapping, + strip_prefix="model.", + ) EntryClass = Qwen3ForCausalLM diff --git a/python/sglang/multimodal_gen/runtime/models/encoders/t5.py b/python/sglang/multimodal_gen/runtime/models/encoders/t5.py index 566a4ddbd3..333a152e6a 100644 --- a/python/sglang/multimodal_gen/runtime/models/encoders/t5.py +++ b/python/sglang/multimodal_gen/runtime/models/encoders/t5.py @@ -158,9 +158,6 @@ class T5LayerFF(nn.Module): # T5 has attn_bias and does not use softmax scaling class T5MultiHeadAttention(nn.Module): - def __init__(self) -> None: - super().__init__() - def forward(self, q, k, v, attn_bias=None): b, _, n, c = q.shape attn = torch.einsum("binc,bjnc->bnij", q, k) @@ -509,7 +506,6 @@ class T5Stack(nn.Module): ): super().__init__() self.embed_tokens = embed_tokens - self.is_umt5 = is_umt5 if is_umt5: self.block = nn.ModuleList( [ diff --git a/python/sglang/multimodal_gen/runtime/models/schedulers/flow_match_pair.py b/python/sglang/multimodal_gen/runtime/models/schedulers/flow_match_pair.py index cb26f9fd9a..bfd9a8d985 100644 --- a/python/sglang/multimodal_gen/runtime/models/schedulers/flow_match_pair.py +++ b/python/sglang/multimodal_gen/runtime/models/schedulers/flow_match_pair.py @@ -49,7 +49,6 @@ class FlowMatchScheduler(BaseScheduler): self, num_inference_steps=100, denoising_strength=1.0, - training=False, shift=None, dynamic_shift_len=None, ): @@ -91,17 +90,6 @@ class FlowMatchScheduler(BaseScheduler): if self.train_timesteps is None: self.train_timesteps = self.timesteps self.train_sigmas = self.sigmas - if training: - x = self.timesteps - y = torch.exp( - -2 * ((x - num_inference_steps / 2) / num_inference_steps) ** 2 - ) - y_shifted = y - y.min() - bsmntw_weighing = y_shifted * (num_inference_steps / y_shifted.sum()) - self.linear_timesteps_weights = bsmntw_weighing - self.training = True - else: - self.training = False def scale_model_input(self, sample: torch.Tensor, timestep: int | None = None): return sample @@ -118,14 +106,6 @@ class FlowMatchScheduler(BaseScheduler): prev_sample = sample + model_output * (sigma_ - sigma) return prev_sample - def return_to_timestep(self, timestep, sample, sample_stablized): - if isinstance(timestep, torch.Tensor): - timestep = timestep.cpu() - timestep_id = torch.argmin((self.timesteps - timestep).abs()) - sigma = self.sigmas[timestep_id] - model_output = (sample - sample_stablized) / sigma - return model_output - def add_noise(self, original_samples, noise, timestep): if isinstance(timestep, torch.Tensor): timestep = timestep.cpu() @@ -134,17 +114,6 @@ class FlowMatchScheduler(BaseScheduler): sample = (1 - sigma) * original_samples + sigma * noise return sample - def training_target(self, sample, noise, timestep): - target = noise - sample - return target - - def training_weight(self, timestep): - timestep_id = torch.argmin( - (self.timesteps - timestep.to(self.timesteps.device)).abs() - ) - weights = self.linear_timesteps_weights[timestep_id] - return weights - def calculate_shift( self, image_seq_len, @@ -268,9 +237,9 @@ class FlowMatchPairScheduler(FlowMatchScheduler): self.set_pair_postprocess(_quadratic_perp_bulge_swap) return - if name == "v2a_sequential": + if name in ("v2a_sequential", "a2v_sequential"): - def _v2a(pairs: torch.Tensor): + def _sequential(pairs: torch.Tensor): if ( not isinstance(pairs, torch.Tensor) or pairs.ndim != 2 @@ -281,8 +250,12 @@ class FlowMatchPairScheduler(FlowMatchScheduler): base = pairs[:, 0] seq_half = base[::2] m = int(seq_half.shape[0]) - col0 = torch.cat([seq_half, seq_half[-1:].repeat(m)], dim=0)[:N] - col1 = torch.cat([seq_half[0:1].repeat(m), seq_half], dim=0)[:N] + if name == "v2a_sequential": + col0 = torch.cat([seq_half, seq_half[-1:].repeat(m)], dim=0)[:N] + col1 = torch.cat([seq_half[0:1].repeat(m), seq_half], dim=0)[:N] + else: + col0 = torch.cat([seq_half[0:1].repeat(m), seq_half], dim=0)[:N] + col1 = torch.cat([seq_half, seq_half[-1:].repeat(m)], dim=0)[:N] return torch.stack( [ col0.to(dtype=pairs.dtype, device=pairs.device), @@ -291,60 +264,25 @@ class FlowMatchPairScheduler(FlowMatchScheduler): dim=1, ) - self.set_pair_postprocess(_v2a) + self.set_pair_postprocess(_sequential) return - if name == "a2v_sequential": + if name in ("v2a", "a2v"): - def _a2v(pairs: torch.Tensor): + def _single_modality(pairs: torch.Tensor): if ( not isinstance(pairs, torch.Tensor) or pairs.ndim != 2 or pairs.shape[1] != 2 ): raise ValueError("pairs must be a torch.Tensor of shape [N, 2]") - N = pairs.shape[0] - base = pairs[:, 0] - seq_half = base[::2] - m = int(seq_half.shape[0]) - col0 = torch.cat([seq_half[0:1].repeat(m), seq_half], dim=0)[:N] - col1 = torch.cat([seq_half, seq_half[-1:].repeat(m)], dim=0)[:N] - return torch.stack( - [ - col0.to(dtype=pairs.dtype, device=pairs.device), - col1.to(dtype=pairs.dtype, device=pairs.device), - ], - dim=1, - ) + if name == "v2a": + zeros = torch.zeros_like(pairs[:, 0]) + return torch.stack([zeros, pairs[:, 1]], dim=1) + else: + zeros = torch.zeros_like(pairs[:, 1]) + return torch.stack([pairs[:, 0], zeros], dim=1) - self.set_pair_postprocess(_a2v) - return - if name == "v2a": - - def _v2a_classic(pairs: torch.Tensor): - if ( - not isinstance(pairs, torch.Tensor) - or pairs.ndim != 2 - or pairs.shape[1] != 2 - ): - raise ValueError("pairs must be a torch.Tensor of shape [N, 2]") - zeros = torch.zeros_like(pairs[:, 0]) - return torch.stack([zeros, pairs[:, 1]], dim=1) - - self.set_pair_postprocess(_v2a_classic) - return - if name == "a2v": - - def _a2v_classic(pairs: torch.Tensor): - if ( - not isinstance(pairs, torch.Tensor) - or pairs.ndim != 2 - or pairs.shape[1] != 2 - ): - raise ValueError("pairs must be a torch.Tensor of shape [N, 2]") - zeros = torch.zeros_like(pairs[:, 1]) - return torch.stack([pairs[:, 0], zeros], dim=1) - - self.set_pair_postprocess(_a2v_classic) + self.set_pair_postprocess(_single_modality) return if name == "dual_sigma_shift": visual_shift = float(kwargs.get("visual_shift", self.shift)) diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/alias_free.py b/python/sglang/multimodal_gen/runtime/models/vaes/alias_free.py index b45405c1a2..16f1085b47 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/alias_free.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/alias_free.py @@ -19,21 +19,7 @@ import torch import torch.nn as nn import torch.nn.functional as F -if "sinc" in dir(torch): - sinc = torch.sinc -else: - # This code is adopted from adefossez's julius.core.sinc under the MIT License - # https://adefossez.github.io/julius/julius/core.html - def sinc(x: torch.Tensor): - """ - Implementation of sinc, i.e. sin(pi * x) / (pi * x) - __Warning__: Different to julius.sinc, the input is multiplied by `pi`! - """ - return torch.where( - x == 0, - torch.tensor(1.0, device=x.device, dtype=x.dtype), - torch.sin(math.pi * x) / math.pi / x, - ) +sinc = torch.sinc # This code is adopted from adefossez's julius.lowpass.LowPassFilters under the MIT License @@ -175,8 +161,6 @@ class Activation1d(nn.Module): down_kernel_size: int = 12, ): super().__init__() - self.up_ratio = up_ratio - self.down_ratio = down_ratio self.act = activation self.upsample = UpSample1d(up_ratio, up_kernel_size) self.downsample = DownSample1d(down_ratio, down_kernel_size) diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_kl_qwenimage.py b/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_kl_qwenimage.py index 9bf4d4eeaa..3a729a22cc 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_kl_qwenimage.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_kl_qwenimage.py @@ -951,18 +951,6 @@ class AutoencoderKLQwenImage(ParallelTiledVAE): self.tile_sample_stride_height = 192 self.tile_sample_stride_width = 192 - # Precompute and cache conv counts for encoder and decoder for clear_cache speedup - self._cached_conv_counts = { - "decoder": sum( - isinstance(m, (QwenImageCausalConv3d, SpatialParallelCausalConv3d)) - for m in self.decoder.modules() - ) - if self.decoder is not None - else 0, - "encoder": sum(isinstance(m, QwenImageCausalConv3d) for m in self.encoder.modules()) - if self.encoder is not None - else 0, - } cuda_device = get_local_torch_device() dtype = torch.get_default_dtype() latent_channels = config.arch_config.z_dim diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/common.py b/python/sglang/multimodal_gen/runtime/models/vaes/common.py index ab613cbcf2..29ceb25222 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/common.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/common.py @@ -7,7 +7,6 @@ from functools import lru_cache from math import isqrt, prod from typing import Optional, cast -import numpy as np import torch import torch.distributed as dist from diffusers.models.autoencoders.vae import DiagonalGaussianDistribution @@ -230,6 +229,14 @@ class ParallelTiledVAE(ABC, nn.Module, LayerwiseOffloadableModuleMixin): ) + b[:, :, x, :, :] * (x / blend_extent) return b + def _spatial_tile_geometry(self) -> tuple[int, int, int, int]: + return ( + self.tile_sample_min_height // self.spatial_compression_ratio, + self.tile_sample_min_width // self.spatial_compression_ratio, + self.tile_sample_stride_height // self.spatial_compression_ratio, + self.tile_sample_stride_width // self.spatial_compression_ratio, + ) + def spatial_tiled_encode(self, x: torch.Tensor) -> torch.Tensor: r"""Encode a batch of images using a tiled encoder. @@ -244,18 +251,12 @@ class ParallelTiledVAE(ABC, nn.Module, LayerwiseOffloadableModuleMixin): # latent_height = height // self.spatial_compression_ratio # latent_width = width // self.spatial_compression_ratio - tile_latent_min_height = ( - self.tile_sample_min_height // self.spatial_compression_ratio - ) - tile_latent_min_width = ( - self.tile_sample_min_width // self.spatial_compression_ratio - ) - tile_latent_stride_height = ( - self.tile_sample_stride_height // self.spatial_compression_ratio - ) - tile_latent_stride_width = ( - self.tile_sample_stride_width // self.spatial_compression_ratio - ) + ( + tile_latent_min_height, + tile_latent_min_width, + tile_latent_stride_height, + tile_latent_stride_width, + ) = self._spatial_tile_geometry() blend_height = tile_latent_min_height - tile_latent_stride_height blend_width = tile_latent_min_width - tile_latent_stride_width @@ -293,21 +294,15 @@ class ParallelTiledVAE(ABC, nn.Module, LayerwiseOffloadableModuleMixin): world_size, rank = sp_group.world_size, sp_group.rank_in_group _, _, T, H, W = z.shape - tile_latent_min_height = ( - self.tile_sample_min_height // self.spatial_compression_ratio - ) - tile_latent_min_width = ( - self.tile_sample_min_width // self.spatial_compression_ratio - ) + ( + tile_latent_min_height, + tile_latent_min_width, + tile_latent_stride_height, + tile_latent_stride_width, + ) = self._spatial_tile_geometry() tile_latent_min_num_frames = ( self.tile_sample_min_num_frames // self.temporal_compression_ratio ) - tile_latent_stride_height = ( - self.tile_sample_stride_height // self.spatial_compression_ratio - ) - tile_latent_stride_width = ( - self.tile_sample_stride_width // self.spatial_compression_ratio - ) tile_latent_stride_num_frames = ( self.tile_sample_stride_num_frames // self.temporal_compression_ratio ) @@ -436,18 +431,12 @@ class ParallelTiledVAE(ABC, nn.Module, LayerwiseOffloadableModuleMixin): if world_size <= 1: return self._decode(z) - tile_latent_min_height = ( - self.tile_sample_min_height // self.spatial_compression_ratio - ) - tile_latent_min_width = ( - self.tile_sample_min_width // self.spatial_compression_ratio - ) - tile_latent_stride_height = ( - self.tile_sample_stride_height // self.spatial_compression_ratio - ) - tile_latent_stride_width = ( - self.tile_sample_stride_width // self.spatial_compression_ratio - ) + ( + tile_latent_min_height, + tile_latent_min_width, + tile_latent_stride_height, + tile_latent_stride_width, + ) = self._spatial_tile_geometry() overlap_h = max(0, tile_latent_min_height - tile_latent_stride_height) overlap_w = max(0, tile_latent_min_width - tile_latent_stride_width) halo_h = overlap_h // 2 @@ -574,18 +563,12 @@ class ParallelTiledVAE(ABC, nn.Module, LayerwiseOffloadableModuleMixin): # sample_height = height * self.spatial_compression_ratio # sample_width = width * self.spatial_compression_ratio - tile_latent_min_height = ( - self.tile_sample_min_height // self.spatial_compression_ratio - ) - tile_latent_min_width = ( - self.tile_sample_min_width // self.spatial_compression_ratio - ) - tile_latent_stride_height = ( - self.tile_sample_stride_height // self.spatial_compression_ratio - ) - tile_latent_stride_width = ( - self.tile_sample_stride_width // self.spatial_compression_ratio - ) + ( + tile_latent_min_height, + tile_latent_min_width, + tile_latent_stride_height, + tile_latent_stride_width, + ) = self._spatial_tile_geometry() blend_height = self.tile_sample_min_height - self.tile_sample_stride_height blend_width = self.tile_sample_min_width - self.tile_sample_stride_width @@ -809,16 +792,5 @@ class DiagonalGaussianDistribution: dim=dims, ) - def nll( - self, sample: torch.Tensor, dims: tuple[int, ...] = (1, 2, 3) - ) -> torch.Tensor: - if self.deterministic: - return torch.Tensor([0.0]) - logtwopi = np.log(2.0 * np.pi) - return 0.5 * torch.sum( - logtwopi + self.logvar + torch.pow(sample - self.mean, 2) / self.var, - dim=dims, - ) - def mode(self) -> torch.Tensor: return self.mean diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/cosmos3_avae.py b/python/sglang/multimodal_gen/runtime/models/vaes/cosmos3_avae.py index ed6a5b054c..3d152a466e 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/cosmos3_avae.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/cosmos3_avae.py @@ -193,12 +193,6 @@ class Cosmos3AVAEAudioTokenizer(nn.Module, LayerwiseOffloadableModuleMixin): def temporal_compression_factor(self) -> int: return self.hop_size - def get_latent_num_samples(self, num_audio_samples: int) -> int: - return int(num_audio_samples) // self.hop_size - - def get_audio_num_samples(self, num_latent_samples: int) -> int: - return int(num_latent_samples) * self.hop_size - def _denormalize_latent(self, latent: torch.Tensor) -> torch.Tensor: if self.normalization_type == "tanh": in_dtype = latent.dtype diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/dac.py b/python/sglang/multimodal_gen/runtime/models/vaes/dac.py index 6c52484cc4..e3f1663c0e 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/dac.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/dac.py @@ -370,7 +370,6 @@ class Encoder(nn.Module): # Wrap black into nn.Sequential self.block = nn.Sequential(*self.block) - self.enc_dim = d_model def forward(self, x): return self.block(x) diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/hunyuan3d_vae.py b/python/sglang/multimodal_gen/runtime/models/vaes/hunyuan3d_vae.py index 8ad0a992a6..6bba427804 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/hunyuan3d_vae.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/hunyuan3d_vae.py @@ -209,7 +209,6 @@ class QKVMultiheadCrossAttention(nn.Module): ): super().__init__() self.heads = heads - self.n_data = n_data self.q_norm = ( norm_layer(width // heads, elementwise_affine=True, eps=1e-6) if qk_norm @@ -255,7 +254,6 @@ class MultiheadCrossAttention(nn.Module): kv_cache: bool = False, ): super().__init__() - self.n_data = n_data self.width = width self.heads = heads self.data_width = width if data_width is None else data_width @@ -338,7 +336,6 @@ class QKVMultiheadAttention(nn.Module): ): super().__init__() self.heads = heads - self.n_ctx = n_ctx self.q_norm = ( norm_layer(width // heads, elementwise_affine=True, eps=1e-6) if qk_norm @@ -381,7 +378,6 @@ class MultiheadAttention(nn.Module): drop_path_rate: float = 0.0, ): super().__init__() - self.n_ctx = n_ctx self.width = width self.heads = heads self.c_qkv = nn.Linear(width, width * 3, bias=qkv_bias) @@ -450,7 +446,6 @@ class Transformer(nn.Module): drop_path_rate: float = 0.0, ): super().__init__() - self.n_ctx = n_ctx self.width = width self.layers = layers self.resblocks = nn.ModuleList( @@ -512,7 +507,6 @@ class CrossAttentionDecoder(nn.Module): if self.enable_ln_post: self.ln_post = nn.LayerNorm(width) self.output_proj = nn.Linear(width, out_channels) - self.label_type = label_type def set_cross_attention_processor(self, processor): self.cross_attn_decoder.attn.attention.attn_processor = processor diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/ltx_2_3_condition_encoder.py b/python/sglang/multimodal_gen/runtime/models/vaes/ltx_2_3_condition_encoder.py index 4bd41d8aaa..1c23eed5d8 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/ltx_2_3_condition_encoder.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/ltx_2_3_condition_encoder.py @@ -131,7 +131,6 @@ class LTX23VideoConditionEncoder( latent_log_var = str(vae_config.get("latent_log_var", "uniform")) self.patch_size = patch_size - self.latency_channels = latent_channels self.latent_log_var = latent_log_var self.per_channel_statistics = LTX23PerChannelStatistics(latent_channels) diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/ltx_2_audio.py b/python/sglang/multimodal_gen/runtime/models/vaes/ltx_2_audio.py index 5d0bbfd997..7011ec5c55 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/ltx_2_audio.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/ltx_2_audio.py @@ -349,7 +349,6 @@ class LTX2AudioAudioPatchifier: ): self.hop_length = hop_length self.sample_rate = sample_rate - self.audio_latent_downsample_factor = audio_latent_downsample_factor self.is_causal = is_causal self._patch_size = (1, patch_size, patch_size) @@ -413,7 +412,6 @@ class LTX2AudioEncoder(nn.Module): base_block_channels = base_channels base_resolution = resolution - self.z_shape = (1, latent_channels, base_resolution, base_resolution) if self.causality_axis is not None: self.conv_in = LTX2AudioCausalConv2d( @@ -594,7 +592,6 @@ class LTX2AudioDecoder(nn.Module): base_block_channels = base_channels * self.channel_multipliers[-1] base_resolution = resolution // (2 ** (self.num_resolutions - 1)) - self.z_shape = (1, latent_channels, base_resolution, base_resolution) if self.causality_axis is not None: self.conv_in = LTX2AudioCausalConv2d( diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/ltx_2_vae.py b/python/sglang/multimodal_gen/runtime/models/vaes/ltx_2_vae.py index 9905f4aa81..a45a50df82 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/ltx_2_vae.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/ltx_2_vae.py @@ -122,9 +122,7 @@ class LTX2VideoCausalConv3d(nn.Module): def _weight_is_channels_last_3d(self) -> bool: w = self.conv.weight - return hasattr(torch, "channels_last_3d") and _is_channels_last_3d_stride( - tuple(w.size()), tuple(w.stride()) - ) + return _is_channels_last_3d_stride(tuple(w.size()), tuple(w.stride())) def _causal_temporal_pad_channels_last( self, @@ -1769,17 +1767,10 @@ class AutoencoderKLLTX2Video(ParallelTiledVAE): # intermediate tiles together, the memory requirement can be lowered. self.use_tiling = False - # When decoding temporally long video latents, the memory requirement is very high. By decoding latent frames - # at a fixed frame batch size (based on `self.num_latent_frames_batch_sizes`), the memory requirement can be lowered. + # Temporal tiling bounds memory when encoding or decoding long videos. self.use_framewise_encoding = False self.use_framewise_decoding = False - # This can be configured based on the amount of GPU memory available. - # `16` for sample frames and `2` for latent frames are sensible defaults for consumer GPUs. - # Setting it to higher values results in higher memory usage. - self.num_sample_frames_batch_size = 16 - self.num_latent_frames_batch_size = 2 - # The minimal tile height and width for spatial tiling to be used self.tile_sample_min_height = 512 self.tile_sample_min_width = 512 diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_audio_vae/audio_vae.py b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_audio_vae/audio_vae.py index c1ec122087..77084a69fd 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_audio_vae/audio_vae.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_audio_vae/audio_vae.py @@ -222,7 +222,6 @@ class Encoder(nn.Module): # Wrap black into nn.Sequential self.block = nn.Sequential(*self.block) - self.enc_dim = d_model def forward(self, x): return self.block(x) diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_audio_vae/bigvgan.py b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_audio_vae/bigvgan.py index 14a720deb0..a8fbd8b582 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_audio_vae/bigvgan.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_audio_vae/bigvgan.py @@ -49,7 +49,6 @@ class SnakeBeta(nn.Module): self.alpha.requires_grad = alpha_trainable self.beta.requires_grad = alpha_trainable - self.no_div_by_zero = 0.000000001 def forward(self, x): alpha = self.alpha.unsqueeze(0).unsqueeze(-1) diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/klvae.py b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/klvae.py index 3f11f2684a..eab7ee373a 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/klvae.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/klvae.py @@ -1248,7 +1248,6 @@ class AutoencoderKLLegacy(AutoencoderKL): self.use_3d_conv = use_3d_conv self.causal_encoder = causal_encoder self.causal_decoder = causal_decoder - self.slidedec = self.causal_encoder and not self.causal_decoder # some registered parameters for simplicity self.vae_ratio = int(np.cumprod(space_down)[-1]) diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/vae_cnn.py b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/vae_cnn.py index 1f72c67f70..9de936e653 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/vae_cnn.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/vae_cnn.py @@ -182,8 +182,6 @@ class EncoderFCN3D(nn.Module): else: self.num_res_blocks = num_res_blocks - self.space_down_factors = space_down - self.time_down_factors = time_down self.in_channels = in_channels self.use_fused_norm = ( diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/wanvae.py b/python/sglang/multimodal_gen/runtime/models/vaes/wanvae.py index 94adf90130..d8e602157b 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/wanvae.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/wanvae.py @@ -79,7 +79,7 @@ first_chunk = contextvars.ContextVar("first_chunk", default=None) def _channels_last_3d_supported_by_platform() -> bool: - return hasattr(torch, "channels_last_3d") and ( + return ( current_platform.is_cuda() or current_platform.is_rocm() or current_platform.is_xpu() diff --git a/python/sglang/multimodal_gen/runtime/models/vlas/pi05_core.py b/python/sglang/multimodal_gen/runtime/models/vlas/pi05_core.py index 791b3125fc..249473b949 100644 --- a/python/sglang/multimodal_gen/runtime/models/vlas/pi05_core.py +++ b/python/sglang/multimodal_gen/runtime/models/vlas/pi05_core.py @@ -117,7 +117,6 @@ class PiGemmaRMSNorm(nn.Module): super().__init__() self.eps = eps self.dim = dim - self.cond_dim = cond_dim if cond_dim is None: self.weight = nn.Parameter(torch.zeros(dim)) self.dense = None @@ -1212,21 +1211,6 @@ class Pi05CoreModel(nn.Module): persistent=False, ) - def retain_runtime_components( - self, - role: Literal["all", "prefix", "action", "idle"], - ) -> None: - if role == "all": - return - if role in ("prefix", "idle"): - self.paligemma_with_expert.gemma_expert = None - self.action_in_proj = None - self.action_out_proj = None - self.time_mlp_in = None - self.time_mlp_out = None - if role in ("action", "idle"): - self.paligemma_with_expert.paligemma = None - def prepare_attention_masks_4d( self, att_2d_masks: torch.Tensor, diff --git a/python/sglang/multimodal_gen/runtime/models/vocoder/ltx_2_vocoder.py b/python/sglang/multimodal_gen/runtime/models/vocoder/ltx_2_vocoder.py index 7e604ca5d0..ab343cb8ad 100644 --- a/python/sglang/multimodal_gen/runtime/models/vocoder/ltx_2_vocoder.py +++ b/python/sglang/multimodal_gen/runtime/models/vocoder/ltx_2_vocoder.py @@ -247,7 +247,6 @@ class ResBlock(nn.Module): padding_mode: str = "same", ): super().__init__() - self.dilations = dilations self.negative_slope = leaky_relu_negative_slope self.convs1 = nn.ModuleList( diff --git a/python/sglang/multimodal_gen/runtime/pipelines/diffusers_pipeline.py b/python/sglang/multimodal_gen/runtime/pipelines/diffusers_pipeline.py index f2cac3ba7d..34a9a4181e 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines/diffusers_pipeline.py +++ b/python/sglang/multimodal_gen/runtime/pipelines/diffusers_pipeline.py @@ -721,9 +721,6 @@ class DiffusersPipeline(ComposedPipelineBase): stage=DiffusersExecutionStage(self.diffusers_pipe), ) - def initialize_pipeline(self, server_args: ServerArgs) -> None: - pass - def post_init(self) -> None: """Post initialization hook.""" if self.post_init_called: diff --git a/python/sglang/multimodal_gen/runtime/pipelines/ltx_2_pipeline.py b/python/sglang/multimodal_gen/runtime/pipelines/ltx_2_pipeline.py index 6a750db23d..aa1ee266bf 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines/ltx_2_pipeline.py +++ b/python/sglang/multimodal_gen/runtime/pipelines/ltx_2_pipeline.py @@ -718,7 +718,6 @@ class LTX2TwoStagePipeline(_BaseLTX2Pipeline): self._distilled_lora_path = distilled_lora_path self._stage1_lora_path = server_args.lora_path self._stage1_lora_scale = float(server_args.lora_scale) - self._active_lora_phase = None self._active_lora_signature = None self._use_premerged_stage2_transformer = False # set when original mode merges stage-1 distilled LoRA into the DiT base @@ -808,7 +807,6 @@ class LTX2TwoStagePipeline(_BaseLTX2Pipeline): self._stage1_distilled_in_base = True self._stage1_distilled_base_strength = strength - self._active_lora_phase = "stage1" self._active_lora_signature = None logger.info( "Merged LTX-2 stage-1 distilled LoRA (strength=%.4f) into the DiT base; " @@ -963,7 +961,6 @@ class LTX2TwoStagePipeline(_BaseLTX2Pipeline): # A pre-distilled DiT has no LoRA to switch to and runs the same # weights in both stages. Guarding here covers every caller. if self._distilled_lora_path is None: - self._active_lora_phase = phase return distilled_lora_strength = self._get_stage_distilled_lora_strength(phase, batch) phase_signature = (phase, distilled_lora_strength) @@ -972,7 +969,6 @@ class LTX2TwoStagePipeline(_BaseLTX2Pipeline): if self._stage1_distilled_in_base: if self._switch_lora_phase_base_merged(phase, distilled_lora_strength): - self._active_lora_phase = phase self._active_lora_signature = phase_signature return # Base was restored (stage-1 strength override); fall through to the @@ -981,7 +977,6 @@ class LTX2TwoStagePipeline(_BaseLTX2Pipeline): if self._ltx2_residency.enter_phase( phase ) and self._can_short_circuit_lora_switch(phase, batch): - self._active_lora_phase = phase self._active_lora_signature = phase_signature return @@ -1015,7 +1010,6 @@ class LTX2TwoStagePipeline(_BaseLTX2Pipeline): # two-stage pipeline immediately. self.deactivate_lora_weights(target="transformer") - self._active_lora_phase = phase self._active_lora_signature = phase_signature def create_pipeline_stages(self, server_args: ServerArgs): diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/base.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/base.py index 91d475b20a..3a9851a24d 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/base.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/base.py @@ -157,21 +157,12 @@ class PipelineStage(StageDedupMixin, ABC): # Default implementation - no verification return VerificationResult() - def maybe_free_model_hooks(self): - pass - def load_model(self): """ Load the model for the stage. """ pass - def offload_model(self): - """ - Offload the model for the stage. - """ - pass - def iter_sequential_requests( self, batch: Req, server_args: ServerArgs ) -> Iterator[Req]: @@ -384,15 +375,6 @@ class PipelineStage(StageDedupMixin, ABC): current_platform.device_type, ) - def set_logging(self, enable: bool): - """ - Enable or disable logging for this stage. - - Args: - enable: Whether to enable logging. - """ - self._enable_logging = enable - def __call__( self, batch: Req, diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising.py index e2f351037e..7de0fadc7d 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising.py @@ -377,7 +377,6 @@ class DenoisingStage(PipelineStage, RolloutDenoisingMixin): self.transformer_2 = transformer_2 # cache-dit state (for delayed mounting and idempotent control) self._cache_dit_enabled = False - self._cached_num_steps = None # Per-request Cache-DiT overrides for the batch being executed # (stashed by _maybe_enable_cache_dit; read by the config builders). self._cache_dit_request_overrides: dict[str, Any] = {} @@ -431,7 +430,6 @@ class DenoisingStage(PipelineStage, RolloutDenoisingMixin): # misc self.profiler = None - self._is_warmed_up = False self._extra_func_kwarg_names_cache: dict[int, tuple[bool, frozenset[str]]] = {} def _infer_transformer_attention_backend(self) -> AttentionBackendEnum | None: @@ -850,7 +848,6 @@ class DenoisingStage(PipelineStage, RolloutDenoisingMixin): for transformer in filter(None, [self.transformer, self.transformer_2]): disable_cache_on_transformer(transformer) self._cache_dit_enabled = False - self._cached_num_steps = None self._cache_dit_active_key = None def _cache_dit_secondary_uses_primary_config(self) -> bool: @@ -1206,7 +1203,6 @@ class DenoisingStage(PipelineStage, RolloutDenoisingMixin): ) self._cache_dit_enabled = True - self._cached_num_steps = num_inference_steps self._cache_dit_active_key = desired_key @lru_cache(maxsize=8) diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/latent_preparation.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/latent_preparation.py index 6664bf01a5..88c1e2523f 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/latent_preparation.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/latent_preparation.py @@ -28,6 +28,29 @@ from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger logger = init_logger(__name__) +def verify_latent_preparation_inputs(batch: Req) -> VerificationResult: + """Input contract shared by generic and model-specific latent preparation.""" + result = VerificationResult() + result.add_check( + "prompt_or_embeds", + None, + lambda _: ( + V.string_or_list_strings(batch.prompt) + or V.list_not_empty(batch.prompt_embeds) + ), + ) + result.add_check("prompt_embeds", batch.prompt_embeds, V.list_of_tensors) + result.add_check( + "num_videos_per_prompt", batch.num_outputs_per_prompt, V.positive_int + ) + result.add_check("generator", batch.generator, V.generator_or_list_generators) + result.add_check("num_frames", batch.num_frames, V.positive_int) + result.add_check("height", batch.height, V.positive_int) + result.add_check("width", batch.width, V.positive_int) + result.add_check("latents", batch.latents, V.none_or_tensor) + return result + + @dataclass(frozen=True) class LatentPreparationFingerprint: height: int | None @@ -352,25 +375,7 @@ class LatentPreparationStage(PipelineStage): def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult: """Verify latent preparation stage inputs.""" - result = VerificationResult() - result.add_check( - "prompt_or_embeds", - None, - lambda _: ( - V.string_or_list_strings(batch.prompt) - or V.list_not_empty(batch.prompt_embeds) - ), - ) - result.add_check("prompt_embeds", batch.prompt_embeds, V.list_of_tensors) - result.add_check( - "num_videos_per_prompt", batch.num_outputs_per_prompt, V.positive_int - ) - result.add_check("generator", batch.generator, V.generator_or_list_generators) - result.add_check("num_frames", batch.num_frames, V.positive_int) - result.add_check("height", batch.height, V.positive_int) - result.add_check("width", batch.width, V.positive_int) - result.add_check("latents", batch.latents, V.none_or_tensor) - return result + return verify_latent_preparation_inputs(batch) def verify_output(self, batch: Req, server_args: ServerArgs) -> VerificationResult: """Verify latent preparation stage outputs.""" diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/glm_image.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/glm_image.py index 5455eb4883..638ecad0ef 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/glm_image.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/glm_image.py @@ -906,14 +906,6 @@ class GlmImageBeforeDenoisingStage(PipelineStage): ) self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) - self.default_sample_size = ( - self.transformer.config.sample_size - if hasattr(self, "transformer") - and self.transformer is not None - and hasattr(self.transformer.config, "sample_size") - else 128 - ) - def component_uses( self, server_args: ServerArgs, stage_name: str | None = None ) -> list[ComponentUse]: diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/helios_denoising.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/helios_denoising.py index aa10545429..91ef3fd9ef 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/helios_denoising.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/helios_denoising.py @@ -29,10 +29,7 @@ from sglang.multimodal_gen.runtime.pipelines_core.diffusion_scheduler_utils impo get_or_create_request_scheduler, ) from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req -from sglang.multimodal_gen.runtime.pipelines_core.stages.base import ( - PipelineStage, - StageParallelismType, -) +from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage from sglang.multimodal_gen.runtime.server_args import ServerArgs from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger from sglang.multimodal_gen.runtime.utils.perf_logger import StageProfiler @@ -139,10 +136,6 @@ class HeliosChunkedDenoisingStage(PipelineStage): def role_affinity(self) -> RoleType: return RoleType.DENOISER - @property - def parallelism_type(self): - return StageParallelismType.REPLICATED - def component_uses( self, server_args: ServerArgs, stage_name: str | None = None ) -> list[ComponentUse]: diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/hunyuan3d/shape.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/hunyuan3d/shape.py index 72cf0868c2..a5f7a74462 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/hunyuan3d/shape.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/hunyuan3d/shape.py @@ -252,9 +252,6 @@ class Hunyuan3DShapeBeforeDenoisingStage(PipelineStage): class Hunyuan3DShapeDenoisingStage(DenoisingStage): """Denoising stage for Hunyuan3D shape generation.""" - def __init__(self, transformer: Any, scheduler: Any, **kwargs) -> None: - super().__init__(transformer=transformer, scheduler=scheduler, **kwargs) - def _component_name_for_stage_module(self, module, default_name: str) -> str: if module is self.transformer: return "hy3dshape_model" diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6/decoding.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6/decoding.py index eddc22a675..096265c4a3 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6/decoding.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6/decoding.py @@ -25,9 +25,6 @@ from sglang.multimodal_gen.runtime.server_args import ServerArgs class Kandinsky6DecodingStage(DecodingStage): """Convert video to channel-first for shared VAE decode; attach the decoded audio.""" - def __init__(self, vae, pipeline=None) -> None: - super().__init__(vae=vae, pipeline=pipeline) - def forward(self, batch: Req, server_args: ServerArgs) -> OutputBatch: if batch.latents is None: raise ValueError("latents must be available before Kandinsky6 decoding.") diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/tiling.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/tiling.py index 2a620d0254..b9d9bbb6b4 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/tiling.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/tiling.py @@ -13,7 +13,6 @@ import torch from torch.nn import functional VAE_SPATIAL_FACTOR: int = 16 -VAE_TEMPORAL_FACTOR: int = 4 # trained base resolutions: visual_size -> [(H, W), ...] RESOLUTIONS: dict[int, list[tuple[int, int]]] = { diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/lingbot_video_moe/text_encoding.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/lingbot_video_moe/text_encoding.py index 510269c718..128511c524 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/lingbot_video_moe/text_encoding.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/lingbot_video_moe/text_encoding.py @@ -34,8 +34,6 @@ PROMPT_TEMPLATE = ( "commentary or evaluations:<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n" "<|im_start|>assistant\n" ) -IMG_PROMPT_TEMPLATE = "<|vision_start|><|image_pad|><|vision_end|>" -VIDEO_PROMPT_TEMPLATE = "<|vision_start|><|video_pad|><|vision_end|>" class LingBotVideoTextEncodingStage(TextEncodingStage): diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/request_validation.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/request_validation.py index 707be26ffe..9a7db8e1de 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/request_validation.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/request_validation.py @@ -13,6 +13,7 @@ import math from collections.abc import Mapping, Sequence from typing import Any +from sglang.multimodal_gen.configs.utils import optional_positive_finite_float from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.constants import ( MINIMAX_H3_MAX_DURATION_SECONDS, MINIMAX_H3_MIN_DURATION_SECONDS, @@ -55,17 +56,6 @@ def _require_int(value: Any, path: str) -> int: return value -def _optional_positive_finite_float(value: Any, path: str) -> float | None: - if value is None: - return None - if isinstance(value, bool) or not isinstance(value, (int, float)): - raise ValueError(f"{path} must be a number") - normalized = float(value) - if not math.isfinite(normalized) or normalized <= 0.0: - raise ValueError(f"{path} must be a positive finite number") - return normalized - - def _optional_nonnegative_finite_float(value: Any, path: str) -> float | None: if value is None: return None @@ -358,8 +348,8 @@ def minimax_h3_validate_canonical_request( "conditions": normalized_conditions, "target": normalized_target, } - normalized_flow_shift = _optional_positive_finite_float(flow_shift, "flow_shift") - normalized_audio_flow_shift = _optional_positive_finite_float( + normalized_flow_shift = optional_positive_finite_float(flow_shift, "flow_shift") + normalized_audio_flow_shift = optional_positive_finite_float( audio_flow_shift, "audio_flow_shift" ) if normalized_flow_shift is not None: diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/denoising.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/denoising.py index f09ee81187..f44355ec2b 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/denoising.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/denoising.py @@ -560,7 +560,6 @@ class MiniMaxH3DenoisingStage(DenoisingStage): # would send the next request down the refresh path with nothing # mounted. self._cache_dit_enabled = False - self._cached_num_steps = None self._cache_dit_active_key = None self._minimax_h3_cache_mode = None self._set_cache_dit_input_preservation(False) diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/latent_preparation.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/latent_preparation.py index a3e9f67f1e..593b6e2787 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/latent_preparation.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/latent_preparation.py @@ -5,6 +5,9 @@ import torch from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage +from sglang.multimodal_gen.runtime.pipelines_core.stages.latent_preparation import ( + verify_latent_preparation_inputs, +) from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import ( StageValidators as V, ) @@ -141,25 +144,7 @@ class MiniMaxH3LatentPreparationStage(PipelineStage): } def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult: - result = VerificationResult() - result.add_check( - "prompt_or_embeds", - None, - lambda _: ( - V.string_or_list_strings(batch.prompt) - or V.list_not_empty(batch.prompt_embeds) - ), - ) - result.add_check("prompt_embeds", batch.prompt_embeds, V.list_of_tensors) - result.add_check( - "num_videos_per_prompt", batch.num_outputs_per_prompt, V.positive_int - ) - result.add_check("generator", batch.generator, V.generator_or_list_generators) - result.add_check("num_frames", batch.num_frames, V.positive_int) - result.add_check("height", batch.height, V.positive_int) - result.add_check("width", batch.width, V.positive_int) - result.add_check("latents", batch.latents, V.none_or_tensor) - return result + return verify_latent_preparation_inputs(batch) def verify_output(self, batch: Req, server_args: ServerArgs) -> VerificationResult: result = VerificationResult() diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/mova.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/mova.py index 5eace7fd2f..b1f9c39e06 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/mova.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/mova.py @@ -168,7 +168,6 @@ class MOVADenoisingStage(PipelineStage): self.dual_tower_bridge = dual_tower_bridge self.scheduler = scheduler self._cache_dit_enabled = False - self._cached_num_steps = None self._torch_compiled = False def component_uses( diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/qwen_image_layered.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/qwen_image_layered.py index 2f98078f7c..d983847241 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/qwen_image_layered.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/qwen_image_layered.py @@ -140,7 +140,6 @@ colors, textures, atmosphere, and so on\n - Identify the text clearly visible in explanation, and highlight it in the caption with quotation marks\n3. Maintain authenticity and accuracy:\n - Avoid generalizations\n - Describe all visible information in the image, while do not add information not explicitly shown in the image\n<|vision_start|><|image_pad|><|vision_end|><|im_end|>\n<|im_start|>assistant\n""" - self.default_sample_size = 128 def component_uses( self, server_args: ServerArgs, stage_name: str | None = None diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/progressive_resolution/wan.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/progressive_resolution/wan.py index 5ca42e926f..00a21c8bb8 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/progressive_resolution/wan.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/progressive_resolution/wan.py @@ -67,21 +67,6 @@ class WanProgressiveDenoisingStage(ProgressiveDenoisingStage): arch, "spatial_compression_ratio", 8 ) - # ------------------------------------------------------------------ - # Resolution-change hook (no-op for Wan T2V) - # ------------------------------------------------------------------ - - def _on_resolution_change( - self, - ctx, - batch: Req, - server_args: ServerArgs, - new_h_pixel: int, - new_w_pixel: int, - ) -> None: - """Wan T2V has no spatial positional embeddings that require updating.""" - pass - # ------------------------------------------------------------------ # Resolution alignment (Wan patch embedding requires even spatial dims) # ------------------------------------------------------------------ diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/realtime/text_encoding.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/realtime/text_encoding.py index 895fc2db23..7f7d6ddaa7 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/realtime/text_encoding.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/realtime/text_encoding.py @@ -38,22 +38,6 @@ def _normalize_prompt_value( return value -def _copy_tensor_list( - value: list[torch.Tensor] | None, -) -> list[torch.Tensor] | None: - if value is None: - return None - return list(value) - - -def _copy_seq_lens( - value: list[list[int]] | None, -) -> list[list[int]] | None: - if value is None: - return None - return [list(seq_lens) for seq_lens in value] - - _TEXT_CACHE_TENSOR_LIST_FIELDS = ( "prompt_embeds", "pooled_embeds", @@ -77,9 +61,11 @@ _TEXT_CACHE_DEFAULT_EMPTY_LIST_FIELDS = { def _copy_text_cache_field(name: str, value): + if value is None: + return None if name in _TEXT_CACHE_SEQ_LENS_FIELDS: - return _copy_seq_lens(value) - return _copy_tensor_list(value) + return [list(seq_lens) for seq_lens in value] + return list(value) class RealtimeTextState(BaseRealtimeState): diff --git a/python/sglang/multimodal_gen/runtime/platforms/cuda.py b/python/sglang/multimodal_gen/runtime/platforms/cuda.py index f713786715..19cd076e75 100644 --- a/python/sglang/multimodal_gen/runtime/platforms/cuda.py +++ b/python/sglang/multimodal_gen/runtime/platforms/cuda.py @@ -590,15 +590,6 @@ class CudaPlatformBase(Platform): def get_device_capability(cls, device_id: int = 0) -> DeviceCapability | None: raise NotImplementedError - @classmethod - def get_device_name(cls, device_id: int = 0) -> str: - raise NotImplementedError - - @classmethod - @lru_cache(maxsize=1) - def get_device_total_memory(cls, device_id: int = 0) -> int: - raise NotImplementedError - @classmethod def is_async_output_supported(cls, enforce_eager: bool | None) -> bool: if enforce_eager: diff --git a/python/sglang/multimodal_gen/runtime/platforms/interface.py b/python/sglang/multimodal_gen/runtime/platforms/interface.py index 2de23d02f7..5026ffacb4 100644 --- a/python/sglang/multimodal_gen/runtime/platforms/interface.py +++ b/python/sglang/multimodal_gen/runtime/platforms/interface.py @@ -536,8 +536,3 @@ class Platform: def tensor_on_device(self, t: torch.Tensor) -> bool: """Check if a tensor is on the current platform's device.""" return t.is_cuda - - -class UnspecifiedPlatform(Platform): - _enum = PlatformEnum.UNSPECIFIED - device_type = "" diff --git a/python/sglang/multimodal_gen/runtime/platforms/mps.py b/python/sglang/multimodal_gen/runtime/platforms/mps.py index e4b25a4aaf..31f793cb67 100644 --- a/python/sglang/multimodal_gen/runtime/platforms/mps.py +++ b/python/sglang/multimodal_gen/runtime/platforms/mps.py @@ -10,7 +10,6 @@ from sglang.multimodal_gen.runtime.platforms import ( Platform, PlatformEnum, ) -from sglang.multimodal_gen.runtime.platforms.interface import DeviceCapability from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger # SPDX-License-Identifier: Apache-2.0 @@ -44,10 +43,6 @@ class MpsPlatform(Platform): def set_device(cls, device: torch.device) -> None: pass - @classmethod - def get_device_capability(cls, device_id: int = 0) -> DeviceCapability | None: - return None - @classmethod def get_device_name(cls, device_id: int = 0) -> str: return "Apple Silicon MPS" @@ -116,11 +111,6 @@ class MpsPlatform(Platform): "sglang.multimodal_gen.runtime.layers.attention.backends.sdpa.SDPABackend" ) - @classmethod - def get_device_communicator_cls(cls) -> str: - # Use base communicator for MPS - return "sglang.multimodal_gen.runtime.distributed.device_communicators.base_device_communicator.DeviceCommunicatorBase" - @classmethod def get_all_to_all_communicator_cls(cls) -> str: return ( diff --git a/python/sglang/multimodal_gen/runtime/platforms/musa.py b/python/sglang/multimodal_gen/runtime/platforms/musa.py index 8983594558..af8ad1ce53 100644 --- a/python/sglang/multimodal_gen/runtime/platforms/musa.py +++ b/python/sglang/multimodal_gen/runtime/platforms/musa.py @@ -84,15 +84,6 @@ class MusaPlatformBase(Platform): def get_device_capability(cls, device_id: int = 0) -> DeviceCapability | None: raise NotImplementedError - @classmethod - def get_device_name(cls, device_id: int = 0) -> str: - raise NotImplementedError - - @classmethod - @lru_cache(maxsize=1) - def get_device_total_memory(cls, device_id: int = 0) -> int: - raise NotImplementedError - @classmethod def is_async_output_supported(cls, enforce_eager: bool | None) -> bool: if enforce_eager: diff --git a/python/sglang/multimodal_gen/runtime/platforms/npu.py b/python/sglang/multimodal_gen/runtime/platforms/npu.py index 032cf71f43..11aaa2d082 100644 --- a/python/sglang/multimodal_gen/runtime/platforms/npu.py +++ b/python/sglang/multimodal_gen/runtime/platforms/npu.py @@ -10,7 +10,6 @@ import torch from sglang.multimodal_gen import envs from sglang.multimodal_gen.runtime.platforms.interface import ( AttentionBackendEnum, - DeviceCapability, Platform, PlatformEnum, ) @@ -53,10 +52,6 @@ class NPUPlatformBase(Platform): def get_local_torch_device(cls) -> torch.device: return torch.device(f"npu:{envs.LOCAL_RANK}") - @classmethod - def get_device_capability(cls, device_id: int = 0) -> DeviceCapability: - return None - @classmethod def get_device_name(cls, device_id: int = 0) -> str: return str(torch.npu.get_device_name(device_id)) @@ -115,10 +110,6 @@ class NPUPlatformBase(Platform): return free_gpu_memory / (1 << 30) - @classmethod - def log_warnings(cls) -> None: - pass - @classmethod def get_current_memory_usage( cls, device: torch.types.Device | None = None diff --git a/python/sglang/multimodal_gen/runtime/platforms/rocm.py b/python/sglang/multimodal_gen/runtime/platforms/rocm.py index 654fc52ac9..d617b41ebd 100644 --- a/python/sglang/multimodal_gen/runtime/platforms/rocm.py +++ b/python/sglang/multimodal_gen/runtime/platforms/rocm.py @@ -78,10 +78,6 @@ class RocmPlatform(Platform): return False return True - @classmethod - def log_warnings(cls) -> None: - pass # ROCm-specific warnings can be added here - @classmethod def get_current_memory_usage(cls, device: torch.device | None = None) -> float: torch.cuda.reset_peak_memory_stats(device) diff --git a/python/sglang/multimodal_gen/runtime/platforms/xpu.py b/python/sglang/multimodal_gen/runtime/platforms/xpu.py index 5fa2d52e5d..d893a1cf73 100644 --- a/python/sglang/multimodal_gen/runtime/platforms/xpu.py +++ b/python/sglang/multimodal_gen/runtime/platforms/xpu.py @@ -62,11 +62,6 @@ class XpuPlatform(Platform): return False return True - @classmethod - def log_warnings(cls) -> None: - """Log any XPU-specific warnings.""" - pass - @classmethod def get_current_memory_usage( cls, device: torch.types.Device | None = None @@ -186,12 +181,6 @@ class XpuPlatform(Platform): "sglang.multimodal_gen.runtime.layers.attention.backends.sdpa.SDPABackend" ) - @classmethod - def get_device_communicator_cls(cls) -> str: - """Get device communicator class for Intel XPU distributed communication.""" - # Use base communicator for now; can be updated to use oneCCL-based communicator - return "sglang.multimodal_gen.runtime.distributed.device_communicators.base_device_communicator.DeviceCommunicatorBase" - @classmethod def get_all_to_all_communicator_cls(cls) -> str: return ( diff --git a/python/sglang/multimodal_gen/runtime/post_training/sp_utils.py b/python/sglang/multimodal_gen/runtime/post_training/sp_utils.py index d05ad0924c..64c1c997e3 100644 --- a/python/sglang/multimodal_gen/runtime/post_training/sp_utils.py +++ b/python/sglang/multimodal_gen/runtime/post_training/sp_utils.py @@ -11,7 +11,6 @@ from sglang.multimodal_gen.runtime.distributed import ( ) from sglang.multimodal_gen.runtime.distributed.communication_op import ( sequence_model_parallel_all_gather, - sequence_model_parallel_all_reduce, ) @@ -36,14 +35,6 @@ def gather_stacked_latents_for_sp( return gathered_flat_inputs.unflatten(0, (bsz, t_steps)) -def all_reduce_if_sp_sharded(batch, tensor: torch.Tensor) -> torch.Tensor: - if not should_do_sp_collective(batch): - return tensor - tensor = tensor.to(get_local_torch_device()) - sequence_model_parallel_all_reduce(tensor) - return tensor - - def all_gather_if_sp_sharded(batch, x: torch.Tensor, dim: int = 0) -> torch.Tensor: if not should_do_sp_collective(batch): return x diff --git a/python/sglang/multimodal_gen/runtime/postprocess/realesrgan_upscaler.py b/python/sglang/multimodal_gen/runtime/postprocess/realesrgan_upscaler.py index 31ea6b5b48..1a8319062a 100644 --- a/python/sglang/multimodal_gen/runtime/postprocess/realesrgan_upscaler.py +++ b/python/sglang/multimodal_gen/runtime/postprocess/realesrgan_upscaler.py @@ -77,12 +77,7 @@ class SRVGGNetCompact(nn.Module): act_type: str = "prelu", ): super().__init__() - self.num_in_ch = num_in_ch - self.num_out_ch = num_out_ch - self.num_feat = num_feat - self.num_conv = num_conv self.upscale = upscale - self.act_type = act_type self.body = nn.ModuleList() # first conv @@ -302,9 +297,6 @@ class UpscalerModel: def dtype(self) -> torch.dtype: return next(self.net.parameters()).dtype - def _copy_input_to_device(self, frames: np.ndarray) -> torch.Tensor: - return torch.from_numpy(frames).to(self.device) - def _preprocess_input_tensor(self, imgs_t: torch.Tensor) -> torch.Tensor: imgs_t = imgs_t.permute(0, 3, 1, 2).to(dtype=self.dtype).mul_(1.0 / 255.0) if self.device.type == "cuda": @@ -316,10 +308,6 @@ class UpscalerModel: out = out.permute(0, 2, 3, 1).clamp(0.0, 1.0).mul_(255.0) return out.to(torch.uint8).contiguous() - @staticmethod - def _copy_output_to_host(out: torch.Tensor) -> np.ndarray: - return out.cpu().numpy() - def _start_cuda_timer(self): if self.device.type != "cuda": return None @@ -463,7 +451,7 @@ class UpscalerModel: start_time = time.perf_counter() h2d_timer = self._start_cuda_timer() - imgs_t = self._copy_input_to_device(imgs) + imgs_t = torch.from_numpy(imgs).to(self.device) self._stop_cuda_timer(h2d_timer) h2d_wall_duration_s = time.perf_counter() - start_time @@ -501,7 +489,7 @@ class UpscalerModel: start_time = time.perf_counter() output_d2h_timer = self._start_cuda_timer() - out_np = self._copy_output_to_host(out) + out_np = out.cpu().numpy() self._stop_cuda_timer(output_d2h_timer) output_d2h_wall_duration_s = time.perf_counter() - start_time diff --git a/python/sglang/multimodal_gen/runtime/postprocess/rife_interpolator.py b/python/sglang/multimodal_gen/runtime/postprocess/rife_interpolator.py index 04c9831c00..f364b371a0 100644 --- a/python/sglang/multimodal_gen/runtime/postprocess/rife_interpolator.py +++ b/python/sglang/multimodal_gen/runtime/postprocess/rife_interpolator.py @@ -350,7 +350,6 @@ class FrameInterpolator: def __init__(self, model_path: Optional[str] = None): self._model_path = model_path - self._resolved_path: Optional[str] = None def _ensure_model_loaded(self) -> Model: """Load RIFE model weights. @@ -368,8 +367,6 @@ class FrameInterpolator: # Resolve: local path pass-through, HF repo ID → download & cache model_path = maybe_download_model(model_path) - self._resolved_path = model_path - if model_path in _MODEL_CACHE: return _MODEL_CACHE[model_path] diff --git a/python/sglang/multimodal_gen/runtime/realtime/video.py b/python/sglang/multimodal_gen/runtime/realtime/video.py index 2c2786ee59..5782ed9e7a 100644 --- a/python/sglang/multimodal_gen/runtime/realtime/video.py +++ b/python/sglang/multimodal_gen/runtime/realtime/video.py @@ -26,7 +26,6 @@ RAW_RGBA_DELTA_GZIP_CONTENT_TYPE = "application/x-raw-rgba-delta-gzip" WEBP_FRAME_CONTENT_TYPE = "image/webp" JPEG_FRAME_CONTENT_TYPE = "image/jpeg" RAW_RGB_CHANNELS = 3 -RAW_RGBA_CHANNELS = 4 _RAW_RGB_DELTA_GZIP_LEVEL = 0 diff --git a/python/sglang/multimodal_gen/runtime/server_args/disagg.py b/python/sglang/multimodal_gen/runtime/server_args/disagg.py index 7453385a1d..0240ca9f57 100644 --- a/python/sglang/multimodal_gen/runtime/server_args/disagg.py +++ b/python/sglang/multimodal_gen/runtime/server_args/disagg.py @@ -3,7 +3,7 @@ from __future__ import annotations -from typing import ClassVar, Literal +from typing import ClassVar from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType from sglang.multimodal_gen.runtime.utils.argparse import FlexibleArgumentParser @@ -74,11 +74,6 @@ class DisaggServerArgsMixin: host, self.scheduler_port + 1, "pool_control_advertised_endpoint" ) - def resolved_role_device(self) -> Literal["cpu", "cuda"]: - if self.disagg_role_device == "auto": - return "cpu" if self.num_gpus <= 0 else "cuda" - return self.disagg_role_device - @classmethod def add_disagg_cli_args(cls, parser: FlexibleArgumentParser) -> None: role_default = ( diff --git a/python/sglang/multimodal_gen/runtime/server_args/server_args.py b/python/sglang/multimodal_gen/runtime/server_args/server_args.py index df8bc672bc..410bed3fa1 100644 --- a/python/sglang/multimodal_gen/runtime/server_args/server_args.py +++ b/python/sglang/multimodal_gen/runtime/server_args/server_args.py @@ -640,13 +640,6 @@ class ServerArgs(DisaggServerArgsMixin): def broker_port(self) -> int: return self.port + 1 - @property - def is_local_mode(self) -> bool: - """ - If no server is running when a generation task begins, 'local_mode' will be enabled: a dedicated server will be launched - """ - return self.host is None or self.port is None - def _adjust_path(self): expand_path_fields(self) self._adjust_save_paths() diff --git a/python/sglang/multimodal_gen/runtime/utils/common.py b/python/sglang/multimodal_gen/runtime/utils/common.py index 8c285c32ab..3de6862fd0 100644 --- a/python/sglang/multimodal_gen/runtime/utils/common.py +++ b/python/sglang/multimodal_gen/runtime/utils/common.py @@ -3,7 +3,6 @@ import ipaddress import logging import os -import platform import socket from functools import lru_cache from typing import Any @@ -268,19 +267,6 @@ def get_zmq_socket( return socket, endpoint -# https://pytorch.org/docs/stable/notes/hip.html#checking-for-hip - - -@lru_cache(maxsize=1) -def is_host_cpu_x86() -> bool: - machine = platform.machine().lower() - return ( - machine in ("x86_64", "amd64", "i386", "i686") - and hasattr(torch, "cpu") - and torch.cpu.is_available() - ) - - # cuda diff --git a/python/sglang/multimodal_gen/runtime/utils/hf_diffusers_utils.py b/python/sglang/multimodal_gen/runtime/utils/hf_diffusers_utils.py index fef29346f0..3bbabdc533 100644 --- a/python/sglang/multimodal_gen/runtime/utils/hf_diffusers_utils.py +++ b/python/sglang/multimodal_gen/runtime/utils/hf_diffusers_utils.py @@ -18,7 +18,6 @@ # ============================================================================== """Utilities for Huggingface Transformers.""" -import contextlib import glob import json import os @@ -400,25 +399,6 @@ def _verify_diffusers_model_complete(path: str) -> bool: ) -_CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = { - # ChatGLMConfig.model_type: ChatGLMConfig, - # DbrxConfig.model_type: DbrxConfig, - # ExaoneConfig.model_type: ExaoneConfig, - # Qwen2_5_VLConfig.model_type: Qwen2_5_VLConfig, -} - -for name, cls in _CONFIG_REGISTRY.items(): - with contextlib.suppress(ValueError): - AutoConfig.register(name, cls) - - -def download_from_hf(model_path: str): - if os.path.exists(model_path): - return model_path - - return snapshot_download(model_path, allow_patterns=["*.json", "*.bin", "*.model"]) - - def get_hf_config( component_model_path: str, trust_remote_code: bool, @@ -435,29 +415,12 @@ def get_hf_config( revision=revision, **kwargs, ) - if config.model_type in _CONFIG_REGISTRY: - config_class = _CONFIG_REGISTRY[config.model_type] - config = config_class.from_pretrained(component_model_path, revision=revision) - # NOTE(HandH1998): Qwen2VL requires `_name_or_path` attribute in `config`. - config._name_or_path = component_model_path if model_override_args: config.update(model_override_args) return config -def get_config( - model: str, - trust_remote_code: bool, - revision: Optional[str] = None, - model_override_args: Optional[dict] = None, - **kwargs, -): - return AutoConfig.from_pretrained( - model, trust_remote_code=trust_remote_code, revision=revision, **kwargs - ) - - def load_dict(file_path): if not os.path.exists(file_path): return {} @@ -604,29 +567,6 @@ def get_diffusers_component_config( return combined_config -# Models don't use the same configuration key for determining the maximum -# context length. Store them here so we can sanely check them. -# NOTE: The ordering here is important. Some models have two of these and we -# have a preference for which value gets used. -CONTEXT_LENGTH_KEYS = [ - "max_sequence_length", - "seq_length", - "max_seq_len", - "model_max_length", - "max_position_embeddings", -] - - -def attach_additional_stop_token_ids(tokenizer): - # Special handling for stop token <|eom_id|> generated by llama 3 tool use. - if "<|eom_id|>" in tokenizer.get_added_vocab(): - tokenizer.additional_stop_token_ids = { - tokenizer.get_added_vocab()["<|eom_id|>"] - } - else: - tokenizer.additional_stop_token_ids = None - - def maybe_download_lora( model_name_or_path: str, local_dir: str | None = None, diff --git a/python/sglang/multimodal_gen/runtime/utils/logging_utils.py b/python/sglang/multimodal_gen/runtime/utils/logging_utils.py index dceecd9fd9..c915ab0f3f 100644 --- a/python/sglang/multimodal_gen/runtime/utils/logging_utils.py +++ b/python/sglang/multimodal_gen/runtime/utils/logging_utils.py @@ -7,7 +7,6 @@ import argparse import contextlib import dataclasses -import datetime import inspect import logging import os @@ -15,16 +14,11 @@ import sys import time from contextlib import contextmanager from enum import Enum -from functools import lru_cache, partial +from functools import lru_cache from logging import Logger from types import MethodType from typing import Any, cast -import sglang.multimodal_gen.envs as envs - -SGLANG_DIFFUSION_LOGGING_LEVEL = envs.SGLANG_DIFFUSION_LOGGING_LEVEL -SGLANG_DIFFUSION_LOGGING_PREFIX = envs.SGLANG_DIFFUSION_LOGGING_PREFIX - # color CYAN = "\033[1;36m" RED = "\033[91m" @@ -32,45 +26,6 @@ GREEN = "\033[92m" YELLOW = "\033[93m" RESET = "\033[0;0m" -_FORMAT = ( - f"{SGLANG_DIFFUSION_LOGGING_PREFIX}%(levelname)s %(asctime)s " - "[%(filename)s: %(lineno)d] %(message)s" -) - -# _FORMAT = "[%(asctime)s] %(message)s" -_DATE_FORMAT = "%m-%d %H:%M:%S" - -DEFAULT_LOGGING_CONFIG = { - "formatters": { - "sgl_diffusion": { - "class": "sglang.multimodal_gen.runtime.utils.logging_utils.ColoredFormatter", - "datefmt": _DATE_FORMAT, - "format": _FORMAT, - }, - }, - "handlers": { - "sgl_diffusion": { - "class": "logging.StreamHandler", - "formatter": "sgl_diffusion", - "level": SGLANG_DIFFUSION_LOGGING_LEVEL, - "stream": "ext://sys.stdout", - }, - }, - "loggers": { - "sgl_diffusion": { - "handlers": ["sgl_diffusion"], - "level": "WARNING", - "propagate": False, - }, - }, - "root": { - "handlers": ["sgl_diffusion"], - "level": "DEBUG", - }, - "version": 1, - "disable_existing_loggers": False, -} - class ColoredFormatter(logging.Formatter): """A logging formatter that adds color to log levels.""" @@ -422,72 +377,6 @@ def _sanitize_for_logging(obj: Any, key_hint: str | None = None) -> Any: return "" -def _trace_calls(log_path, root_dir, frame, event, arg=None): - if event in ["call", "return"]: - # Extract the filename, line number, function name, and the code object - filename = frame.f_code.co_filename - lineno = frame.f_lineno - func_name = frame.f_code.co_name - if not filename.startswith(root_dir): - # only log the functions in the sgl_diffusion root_dir - return - # Log every function call or return - try: - last_frame = frame.f_back - if last_frame is not None: - last_filename = last_frame.f_code.co_filename - last_lineno = last_frame.f_lineno - last_func_name = last_frame.f_code.co_name - else: - # initial frame - last_filename = "" - last_lineno = 0 - last_func_name = "" - with open(log_path, "a") as f: - ts = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f") - if event == "call": - f.write( - f"{ts} Call to" - f" {func_name} in {filename}:{lineno}" - f" from {last_func_name} in {last_filename}:" - f"{last_lineno}\n" - ) - else: - f.write( - f"{ts} Return from" - f" {func_name} in {filename}:{lineno}" - f" to {last_func_name} in {last_filename}:" - f"{last_lineno}\n" - ) - except NameError: - # modules are deleted during shutdown - pass - return partial(_trace_calls, log_path, root_dir) - - -def enable_trace_function_call(log_file_path: str, root_dir: str | None = None): - """ - Enable tracing of every function call in code under `root_dir`. - This is useful for debugging hangs or crashes. - `log_file_path` is the path to the log file. - `root_dir` is the root directory of the code to trace. If None, it is the - sgl_diffusion root directory. - - Note that this call is thread-level, any threads calling this function - will have the trace enabled. Other threads will not be affected. - """ - logger.warning( - "SGLANG_DIFFUSION_TRACE_FUNCTION is enabled. It will record every" - " function executed by Python. This will slow down the code. It " - "is suggested to be used for debugging hang or crashes only." - ) - logger.info("Trace frame log is saved to %s", log_file_path) - if root_dir is None: - # by default, this is the sgl_diffusion root directory - root_dir = os.path.dirname(os.path.dirname(__file__)) - sys.settrace(partial(_trace_calls, log_file_path, root_dir)) - - def set_uvicorn_logging_configs(server_args=None): from uvicorn.config import LOGGING_CONFIG @@ -587,17 +476,6 @@ def get_log_level() -> int: return root.level -def suppress_loggers(loggers_to_suppress: list[str], level: int = logging.WARNING): - original_levels = {} - - for logger_name in loggers_to_suppress: - logger = logging.getLogger(logger_name) - original_levels[logger_name] = logger.level - logger.setLevel(level) - - return original_levels - - def globally_suppress_loggers(): # globally suppress some obsessive loggers target_names = [ diff --git a/python/sglang/multimodal_gen/runtime/utils/mesh3d_utils.py b/python/sglang/multimodal_gen/runtime/utils/mesh3d_utils.py index 2820002861..55919b2585 100644 --- a/python/sglang/multimodal_gen/runtime/utils/mesh3d_utils.py +++ b/python/sglang/multimodal_gen/runtime/utils/mesh3d_utils.py @@ -305,12 +305,10 @@ class MeshRender: self.set_default_texture_resolution(texture_size) self.camera_distance = camera_distance - self.camera_type = camera_type self.bake_angle_thres = 75 self.bake_unreliable_kernel_size = int( (2 / 512) * max(self.default_resolution[0], self.default_resolution[1]) ) - self.bake_mode = bake_mode # Set up camera projection matrix if camera_type == "orth": diff --git a/python/sglang/multimodal_gen/runtime/utils/quantization_utils.py b/python/sglang/multimodal_gen/runtime/utils/quantization_utils.py index cd48f0666c..309fafd7a6 100644 --- a/python/sglang/multimodal_gen/runtime/utils/quantization_utils.py +++ b/python/sglang/multimodal_gen/runtime/utils/quantization_utils.py @@ -438,14 +438,6 @@ def normalize_flat_modelopt_quant_config( return normalized -def _infer_nvfp4_group_size_from_tensors(weight, scale) -> Optional[int]: - """Infer NVFP4 group_size from serialized weight/scale tensor shapes.""" - return _infer_nvfp4_group_size_from_shapes( - getattr(weight, "shape", ()), - getattr(scale, "shape", ()), - ) - - def _infer_nvfp4_group_size_from_shapes(weight_shape, scale_shape) -> Optional[int]: weight_shape = tuple(weight_shape or ()) scale_shape = tuple(scale_shape or ()) @@ -741,7 +733,7 @@ def _canonicalize_modulation_exclude(module_name: str) -> str: return module_name -def _build_nvfp4_config_from_safetensors_files( +def build_nvfp4_config_from_safetensors_list( file_paths: list[str], param_names_mapping_dict: Optional[dict] = None, reverse_param_names_mapping_dict: Optional[dict] = None, @@ -953,32 +945,3 @@ def _build_nvfp4_config_from_safetensors_files( e, ) return None - - -def build_nvfp4_config_from_safetensors( - file_path: str, - param_names_mapping_dict: Optional[dict] = None, - reverse_param_names_mapping_dict: Optional[dict] = None, - fallback_group_size: Optional[int] = None, -) -> Optional[QuantizationConfig]: - """Backward-compatible wrapper for a single safetensors file.""" - return _build_nvfp4_config_from_safetensors_files( - [file_path], - param_names_mapping_dict, - reverse_param_names_mapping_dict, - fallback_group_size, - ) - - -def build_nvfp4_config_from_safetensors_list( - file_paths: list[str], - param_names_mapping_dict: Optional[dict] = None, - reverse_param_names_mapping_dict: Optional[dict] = None, - fallback_group_size: Optional[int] = None, -) -> Optional[QuantizationConfig]: - return _build_nvfp4_config_from_safetensors_files( - file_paths, - param_names_mapping_dict, - reverse_param_names_mapping_dict, - fallback_group_size, - ) diff --git a/python/sglang/multimodal_gen/runtime/utils/vision.py b/python/sglang/multimodal_gen/runtime/utils/vision.py index d3610187ef..2c311cfa23 100644 --- a/python/sglang/multimodal_gen/runtime/utils/vision.py +++ b/python/sglang/multimodal_gen/runtime/utils/vision.py @@ -111,9 +111,7 @@ def load_image( else: # in-memory loading path image = PIL.Image.open(BytesIO(srt_get_image_bytes(image))) - elif isinstance(image, PIL.Image.Image): - image = image - else: + elif not isinstance(image, PIL.Image.Image): raise ValueError( "Incorrect format used for the image. Should be bytes, a URL, a local path, base64/data URL, or a PIL image." ) diff --git a/python/sglang/multimodal_gen/test/single_test_file/test_disagg_server.py b/python/sglang/multimodal_gen/test/single_test_file/test_disagg_server.py index 9e2d818820..8a1017241f 100755 --- a/python/sglang/multimodal_gen/test/single_test_file/test_disagg_server.py +++ b/python/sglang/multimodal_gen/test/single_test_file/test_disagg_server.py @@ -203,7 +203,7 @@ class DisaggCluster: log = _LOG_DIR / f"disagg_{self.name}_server.log" self._logs["server"] = log # Role processes register their transfer work_endpoint with the - # derived value ``tcp://0.0.0.0:`` (see disagg_args.py). The + # derived value ``tcp://0.0.0.0:`` (see server_args/disagg.py). The # server head must advertise the same literal so ``_handle_register``'s # endpoint_to_idx exact-string match succeeds. cmd = [ diff --git a/python/sglang/multimodal_gen/test/test_utils.py b/python/sglang/multimodal_gen/test/test_utils.py index 526bc97d55..df0e4ba14b 100644 --- a/python/sglang/multimodal_gen/test/test_utils.py +++ b/python/sglang/multimodal_gen/test/test_utils.py @@ -13,7 +13,7 @@ import time import wave from dataclasses import dataclass from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import Any from urllib.parse import urljoin import cv2 @@ -30,9 +30,6 @@ from sglang.multimodal_gen.runtime.utils.perf_logger import ( get_diffusion_perf_log_dir, ) -if TYPE_CHECKING: - from sglang.multimodal_gen.test.server.testcase_configs import DiffusionTestCase - logger = init_logger(__name__) # GT is read from @. A given SHA only exists in the repo it was @@ -248,16 +245,6 @@ def is_image_url(image_path: str | Path | None) -> bool: ) -def probe_port(host="127.0.0.1", port=30010, timeout=2.0) -> bool: - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.settimeout(timeout) - try: - s.connect((host, port)) - return True - except OSError: - return False - - def is_in_ci() -> bool: return get_bool_env_var("SGLANG_IS_IN_CI") @@ -311,16 +298,6 @@ def wait_for_server_health( ) -def post_json( - base_url: str, - path: str, - payload: dict, - timeout: float = 300.0, -) -> httpx.Response: - """POST JSON to ```` and return the response.""" - return httpx.post(urljoin(base_url, path), json=payload, timeout=timeout) - - def run_command(command: list[str]) -> bool: """Run a CLI command and return whether it succeeded.""" print(f"Running command: {' '.join(command)}", flush=True) @@ -371,35 +348,6 @@ def query_gpu_mem_used_mib(gpu_index: int = 0, required: bool = False) -> int | return None -def require_gpu_mem_query(gpu_index: int = 0) -> int: - """Same as :func:`query_gpu_mem_used_mib` but asserts availability. - - Raises ``AssertionError`` when ``nvidia-smi`` is unavailable instead of - returning ``None``, so callers can rely on a valid ``int`` result. - """ - mem = query_gpu_mem_used_mib(gpu_index, required=True) - assert mem is not None - return mem - - -def assert_gpu_mem_changed( - label: str, - before_mib: int, - after_mib: int, - min_delta_mib: int, -) -> None: - """Assert that GPU memory changed by at least *min_delta_mib* MiB.""" - delta = abs(after_mib - before_mib) - logger.debug( - f"[MEM] {label}: before={before_mib} MiB after={after_mib} MiB |delta|={delta} MiB" - ) - assert delta >= min_delta_mib, ( - f"GPU memory change too small for '{label}': " - f"|after-before|={delta} MiB < {min_delta_mib} MiB " - f"(before={before_mib} MiB, after={after_mib} MiB)" - ) - - def is_mp4(data: bytes) -> bool: """Check if data represents a valid MP4 file by magic bytes.""" if len(data) < 8: @@ -454,16 +402,6 @@ def get_expected_image_format( return "jpg" # Default -def wait_for_port(host="127.0.0.1", port=30010, deadline=300.0, interval=0.5): - end = time.time() + deadline - last_err = None - while time.time() < end: - if probe_port(host, port, timeout=interval): - return True - time.sleep(interval) - raise TimeoutError(f"Port {host}:{port} not ready. Last error: {last_err}") - - def check_image_size(ut, image, width, height): # check image size ut.assertEqual(image.size, (width, height)) @@ -549,13 +487,6 @@ def validate_image(b64_json: str) -> None: assert is_png(image_bytes) or is_jpeg(image_bytes), "Image must be PNG or JPEG" -def validate_video(b64_json: str) -> None: - """Decode and validate that video is a valid format.""" - video_bytes = base64.b64decode(b64_json) - is_webm = video_bytes[:4] == b"\x1a\x45\xdf\xa3" - assert is_mp4(video_bytes) or is_webm, "Video must be MP4 or WebM" - - def validate_openai_video(video_bytes: bytes) -> None: """Validate that video is MP4 or WebM by magic bytes.""" is_webm = video_bytes.startswith(b"\x1a\x45\xdf\xa3") @@ -1159,18 +1090,6 @@ def compare_audio_with_gt( ) -def get_clip_threshold( - case: "DiffusionTestCase", - metadata: dict[str, Any] | None = None, -) -> float: - """Get CLIP similarity threshold for a consistency test case.""" - return get_consistency_thresholds( - case_id=case.id, - is_video=case.server_args.modality == "video", - metadata=metadata, - ).clip_threshold - - @dataclass class FrameConsistencyMetrics: frame_index: int @@ -1981,21 +1900,6 @@ def load_audio_consistency_gt(case_id: str, num_gpus: int) -> np.ndarray: return loaded_gt -def load_gt_embeddings( - case_id: str, - num_gpus: int, - is_video: bool = False, - output_format: str | None = None, -) -> list[np.ndarray]: - """Load GT images and convert them into CLIP embeddings.""" - return load_consistency_gt( - case_id=case_id, - num_gpus=num_gpus, - is_video=is_video, - output_format=output_format, - ).embeddings - - def gt_exists( case_id: str, num_gpus: int, diff --git a/python/sglang/multimodal_gen/test/unit/test_cache_dit_per_request.py b/python/sglang/multimodal_gen/test/unit/test_cache_dit_per_request.py index bde9d6513a..a3a08c4522 100644 --- a/python/sglang/multimodal_gen/test/unit/test_cache_dit_per_request.py +++ b/python/sglang/multimodal_gen/test/unit/test_cache_dit_per_request.py @@ -122,7 +122,6 @@ class TestPerRequestCacheDitTransitions(unittest.TestCase): self.stage.transformer = object() self.stage.transformer_2 = None self.stage._cache_dit_enabled = False - self.stage._cached_num_steps = None self.stage._cache_dit_request_overrides = {} self.stage._cache_dit_active_key = None diff --git a/python/sglang/multimodal_gen/test/unit/test_server_args.py b/python/sglang/multimodal_gen/test/unit/test_server_args.py index e64ee3d2c2..6688d324ca 100644 --- a/python/sglang/multimodal_gen/test/unit/test_server_args.py +++ b/python/sglang/multimodal_gen/test/unit/test_server_args.py @@ -3322,18 +3322,6 @@ class TestPerRoleParallelism(unittest.TestCase): self.assertEqual(args.get_role_parallelism(RoleType.DENOISER)["tp_size"], 2) self.assertEqual(args.get_role_parallelism(RoleType.DECODER)["sp_degree"], 4) - def test_disagg_args_import_path_matches_server_args_package(self): - from sglang.multimodal_gen.runtime.disaggregation import disagg_args - from sglang.multimodal_gen.runtime.server_args.disagg import ( - DisaggServerArgsMixin, - ) - - self.assertIs(disagg_args.DisaggArgsMixin, DisaggServerArgsMixin) - self.assertIs( - disagg_args.DISAGG_RESULT_PORT_OFFSETS, - DisaggServerArgsMixin.DISAGG_RESULT_PORT_OFFSETS, - ) - def test_gpu_ids_normalize_lists_and_commas(self): args = self._from_dict({"model_path": "/fake", "gpu_ids": ["0,1", "6", "7 8"]})