diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/server_api.py b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/server_api.py
index 944d93ca82..7e90f51fb2 100644
--- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/server_api.py
+++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/server_api.py
@@ -3,14 +3,11 @@ SGLang Diffusion Server API client.
Provides a low-level interface for interacting with SGLang Diffusion HTTP server.
"""
-import base64
-import io
import os
import time
from typing import Any, Dict, Optional
import requests
-from PIL import Image
class SGLDiffusionServerAPI:
@@ -417,38 +414,6 @@ class SGLDiffusionServerAPI:
}
return content_types.get(ext, "image/png")
- def decode_image_from_response(
- self, response_data: Dict[str, Any], index: int = 0
- ) -> Image.Image:
- """
- Decode base64 image from API response.
-
- Args:
- response_data: API response dictionary
- index: Index of the image in the response (default: 0)
-
- Returns:
- PIL Image object
- """
- if "data" not in response_data or not response_data["data"]:
- raise ValueError("No image data in response")
-
- if index >= len(response_data["data"]):
- raise IndexError(f"Image index {index} out of range")
-
- image_data = response_data["data"][index]
- if "b64_json" not in image_data or not image_data["b64_json"]:
- raise ValueError("No base64 image data found")
-
- image_bytes = base64.b64decode(image_data["b64_json"])
- image = Image.open(io.BytesIO(image_bytes))
-
- # Convert to RGB if needed
- if image.mode != "RGB":
- image = image.convert("RGB")
-
- return image
-
def set_lora(
self,
lora_nickname: str,
diff --git a/python/sglang/multimodal_gen/apps/webui/main.py b/python/sglang/multimodal_gen/apps/webui/main.py
index 1391130377..4018c05fba 100644
--- a/python/sglang/multimodal_gen/apps/webui/main.py
+++ b/python/sglang/multimodal_gen/apps/webui/main.py
@@ -1,4 +1,3 @@
-import argparse
import os
from sglang.multimodal_gen.apps.webui.minimax_h3 import (
@@ -21,13 +20,6 @@ from sglang.srt.environ import envs
logger = init_logger(__name__)
-def add_webui_args(parser: argparse.ArgumentParser):
- """Add the arguments for the generate command."""
- parser = ServerArgs.add_cli_args(parser)
- parser = SamplingParams.add_cli_args(parser)
- return parser
-
-
def run_sgl_diffusion_webui(server_args: ServerArgs):
if is_minimax_h3(server_args):
return run_minimax_h3_webui(server_args)
diff --git a/python/sglang/multimodal_gen/configs/models/dits/krea2.py b/python/sglang/multimodal_gen/configs/models/dits/krea2.py
index b330b53642..d103b695d7 100644
--- a/python/sglang/multimodal_gen/configs/models/dits/krea2.py
+++ b/python/sglang/multimodal_gen/configs/models/dits/krea2.py
@@ -65,11 +65,6 @@ class Krea2ArchConfig(DiTArchConfig):
def head_dim(self) -> int:
return self.features // self.heads
- @property
- def in_features_packed(self) -> int:
- """Patch-embed input width: channels * patch**2."""
- return self.channels * self.patch**2
-
@dataclass
class Krea2DitConfig(DiTConfig):
diff --git a/python/sglang/multimodal_gen/configs/models/dits/ltx_2.py b/python/sglang/multimodal_gen/configs/models/dits/ltx_2.py
index 42ed057cda..81b5355c25 100644
--- a/python/sglang/multimodal_gen/configs/models/dits/ltx_2.py
+++ b/python/sglang/multimodal_gen/configs/models/dits/ltx_2.py
@@ -17,12 +17,6 @@ class LTXModelType(Enum):
VideoOnly = "ltx video only model"
AudioOnly = "ltx audio only model"
- def is_video_enabled(self) -> bool:
- return self in (LTXModelType.AudioVideo, LTXModelType.VideoOnly)
-
- def is_audio_enabled(self) -> bool:
- return self in (LTXModelType.AudioVideo, LTXModelType.AudioOnly)
-
class LTX2RopeType(str, Enum):
"""
diff --git a/python/sglang/multimodal_gen/configs/models/encoders/ideogram.py b/python/sglang/multimodal_gen/configs/models/encoders/ideogram.py
index fab9713728..5313a47f67 100644
--- a/python/sglang/multimodal_gen/configs/models/encoders/ideogram.py
+++ b/python/sglang/multimodal_gen/configs/models/encoders/ideogram.py
@@ -31,7 +31,4 @@ class Ideogram4TextEncoderConfig(Qwen3VLConfig):
self.arch_config.ideogram_fp8_weight_only = not use_bitsandbytes
self.arch_config.requires_gpu_resident_text_encoder = use_bitsandbytes
- def finalize_model_arch(self):
- self.post_diffusers_config_update()
-
arch_config: Qwen3VLArchConfig = field(default_factory=Qwen3VLArchConfig)
diff --git a/python/sglang/multimodal_gen/configs/models/vaes/base.py b/python/sglang/multimodal_gen/configs/models/vaes/base.py
index 934a02219b..6afc5d48fb 100644
--- a/python/sglang/multimodal_gen/configs/models/vaes/base.py
+++ b/python/sglang/multimodal_gen/configs/models/vaes/base.py
@@ -16,6 +16,19 @@ AUTO_PARALLEL_DECODE_MODE = "auto"
SPATIAL_SHARD_PARALLEL_DECODE_MODES = ("spatial_shard", "spatial")
+def get_channelwise_decode_scale_and_shift(arch_config, device, dtype):
+ """Return reciprocal std and mean broadcast over BCTHW latents."""
+ scaling_factor = 1.0 / torch.tensor(arch_config.latents_std, device=device).view(
+ 1, arch_config.z_dim, 1, 1, 1
+ ).to(device, dtype)
+ shift_factor = (
+ torch.tensor(arch_config.latents_mean)
+ .view(1, arch_config.z_dim, 1, 1, 1)
+ .to(device, dtype)
+ )
+ return scaling_factor, shift_factor
+
+
@lru_cache(maxsize=8)
def is_spatial_shard_parallel_decode_mode(mode: str) -> bool:
return mode in SPATIAL_SHARD_PARALLEL_DECODE_MODES
diff --git a/python/sglang/multimodal_gen/configs/models/vaes/ltx_video.py b/python/sglang/multimodal_gen/configs/models/vaes/ltx_video.py
index 8e36617289..aa91f063bd 100644
--- a/python/sglang/multimodal_gen/configs/models/vaes/ltx_video.py
+++ b/python/sglang/multimodal_gen/configs/models/vaes/ltx_video.py
@@ -2,9 +2,38 @@
from dataclasses import dataclass, field
from typing import Any, List
+import torch
+
from sglang.multimodal_gen.configs.models.vaes.base import VAEArchConfig, VAEConfig
+def get_ltx_video_decode_scale_and_shift(device, dtype, vae, arch_config):
+ """Invert LTX video normalization: (z - mean) * scaling_factor / std."""
+ latents_mean = getattr(vae, "latents_mean", None)
+ latents_std = getattr(vae, "latents_std", None)
+ scaling_factor = (
+ getattr(getattr(vae, "config", None), "scaling_factor", None)
+ or getattr(vae, "scaling_factor", None)
+ or getattr(arch_config, "scaling_factor", None)
+ or 1.0
+ )
+ if isinstance(scaling_factor, (int, float)) and float(scaling_factor) == 0.0:
+ scaling_factor = 1.0
+
+ if isinstance(latents_mean, torch.Tensor) and isinstance(latents_std, torch.Tensor):
+ latents_mean = latents_mean.to(device=device, dtype=dtype).view(1, -1, 1, 1, 1)
+ latents_std = latents_std.to(device=device, dtype=dtype).view(1, -1, 1, 1, 1)
+ sf = torch.tensor(float(scaling_factor), device=device, dtype=dtype).view(
+ 1, 1, 1, 1, 1
+ )
+ return sf / latents_std, latents_mean
+
+ sf = torch.tensor(float(scaling_factor), device=device, dtype=dtype).view(
+ 1, 1, 1, 1, 1
+ )
+ return sf, None
+
+
@dataclass
class LTXVideoVAEArchConfig(VAEArchConfig):
# Architecture params
diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/base.py b/python/sglang/multimodal_gen/configs/pipeline_configs/base.py
index f126207ae6..7ad38d1ad9 100644
--- a/python/sglang/multimodal_gen/configs/pipeline_configs/base.py
+++ b/python/sglang/multimodal_gen/configs/pipeline_configs/base.py
@@ -5,9 +5,8 @@ import json
import math
import os
from collections.abc import Callable
-from dataclasses import asdict, dataclass, field, fields
+from dataclasses import dataclass, field, fields
from enum import Enum
-from operator import attrgetter
from typing import Any, ClassVar
import numpy as np
@@ -56,6 +55,16 @@ class STA_Mode(str, Enum):
NONE = None
+def pack_latents_2x2(latents, batch_size, num_channels_latents, height, width):
+ latents = latents.view(
+ batch_size, num_channels_latents, height // 2, 2, width // 2, 2
+ )
+ latents = latents.permute(0, 2, 4, 1, 3, 5)
+ return latents.reshape(
+ batch_size, (height // 2) * (width // 2), num_channels_latents * 4
+ )
+
+
def postprocess_text(output: BaseEncoderOutput, _text_inputs) -> torch.tensor:
raise NotImplementedError
@@ -1198,35 +1207,6 @@ class PipelineConfig:
f"Length of text postprocess functions ({len(self.postprocess_text_funcs)}) must be equal to length of text preprocessing functions ({len(self.preprocess_text_funcs)})"
)
- def dump_to_json(self, file_path: str):
- output_dict = {f.name: attrgetter(f.name)(self) for f in fields(self)}
- del_keys = []
- for key, value in output_dict.items():
- if isinstance(value, ModelConfig):
- model_dict = asdict(value)
- # Model Arch Config should be hidden away from the users
- model_dict.pop("arch_config")
- output_dict[key] = model_dict
- elif isinstance(value, tuple) and all(
- isinstance(v, ModelConfig) for v in value
- ):
- model_dicts = []
- for v in value:
- model_dict = asdict(v)
- # Model Arch Config should be hidden away from the users
- model_dict.pop("arch_config")
- model_dicts.append(model_dict)
- output_dict[key] = model_dicts
- elif isinstance(value, tuple) and all(callable(f) for f in value):
- # Skip dumping functions
- del_keys.append(key)
-
- for key in del_keys:
- output_dict.pop(key, None)
-
- with open(file_path, "w") as f:
- json.dump(output_dict, f, indent=2)
-
def load_from_json(self, file_path: str):
with open(file_path) as f:
input_pipeline_dict = json.load(f)
diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/diffusers_generic.py b/python/sglang/multimodal_gen/configs/pipeline_configs/diffusers_generic.py
index b3a3e95721..734f864272 100644
--- a/python/sglang/multimodal_gen/configs/pipeline_configs/diffusers_generic.py
+++ b/python/sglang/multimodal_gen/configs/pipeline_configs/diffusers_generic.py
@@ -67,12 +67,6 @@ class DiffusersGenericPipelineConfig(PipelineConfig):
"""
pass
- def adjust_size(self, width, height, image):
- """
- Pass through - diffusers handles size adjustments.
- """
- return width, height
-
# Static subclasses for each non-default task type.
# These exist so that _get_diffusers_model_info() can swap the task_type without
diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/flux.py b/python/sglang/multimodal_gen/configs/pipeline_configs/flux.py
index 0b344904b4..8d21318f8a 100644
--- a/python/sglang/multimodal_gen/configs/pipeline_configs/flux.py
+++ b/python/sglang/multimodal_gen/configs/pipeline_configs/flux.py
@@ -20,12 +20,12 @@ from sglang.multimodal_gen.configs.models.vaes.flux import Flux2VAEConfig, FluxV
from sglang.multimodal_gen.configs.pipeline_configs.base import (
ImagePipelineConfig,
ModelTaskType,
+ pack_latents_2x2,
shard_rotary_emb_for_sp,
)
from sglang.multimodal_gen.configs.pipeline_configs.hunyuan import (
clip_postprocess_text,
)
-from sglang.multimodal_gen.configs.pipeline_configs.qwen_image import _pack_latents
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
@@ -141,7 +141,9 @@ class FluxPipelineConfig(ImagePipelineConfig):
width = 2 * (batch.width // (self.vae_config.arch_config.vae_scale_factor * 2))
num_channels_latents = self.dit_config.arch_config.in_channels // 4
# pack latents
- return _pack_latents(latents, batch_size, num_channels_latents, height, width)
+ return pack_latents_2x2(
+ latents, batch_size, num_channels_latents, height, width
+ )
def get_pos_prompt_embeds(self, batch):
return batch.prompt_embeds[1]
diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/joy_image.py b/python/sglang/multimodal_gen/configs/pipeline_configs/joy_image.py
index 2e68c4851a..4773d11a30 100644
--- a/python/sglang/multimodal_gen/configs/pipeline_configs/joy_image.py
+++ b/python/sglang/multimodal_gen/configs/pipeline_configs/joy_image.py
@@ -11,6 +11,9 @@ from sglang.multimodal_gen.configs.models import DiTConfig, EncoderConfig, VAECo
from sglang.multimodal_gen.configs.models.dits.joy_image import JoyImageDiTConfig
from sglang.multimodal_gen.configs.models.encoders.qwen3vl import Qwen3VLConfig
from sglang.multimodal_gen.configs.models.vaes import WanVAEConfig
+from sglang.multimodal_gen.configs.models.vaes.base import (
+ get_channelwise_decode_scale_and_shift,
+)
from sglang.multimodal_gen.configs.pipeline_configs.base import (
ImagePipelineConfig,
ModelTaskType,
@@ -249,32 +252,10 @@ class JoyImageEditPipelineConfig(ImagePipelineConfig):
def get_decode_scale_and_shift(
self, device, dtype, vae
) -> Tuple[torch.Tensor, torch.Tensor]:
- """Get VAE denormalization scale and shift.
-
- Args:
- device: Target device
- dtype: Target dtype
- vae: VAE model
-
- Returns:
- Tuple of (scaling_factor, shift_factor)
- """
- vae_arch_config = self.vae_config.arch_config
-
- # Create scale factor: 1.0 / std
- scaling_factor = 1.0 / torch.tensor(
- vae_arch_config.latents_std, device=device
- ).view(1, vae_arch_config.z_dim, 1, 1, 1).to(device, dtype)
-
- # Create shift factor: mean
- shift_factor = (
- torch.tensor(vae_arch_config.latents_mean)
- .view(1, vae_arch_config.z_dim, 1, 1, 1)
- .to(device, dtype)
+ return get_channelwise_decode_scale_and_shift(
+ self.vae_config.arch_config, device, dtype
)
- return scaling_factor, shift_factor
-
def prepare_calculated_size(self, img: Image.Image) -> Tuple[int, int]:
img_h, img_w = img.size[1], img.size[0] # PIL (w,h)
bucket = self.find_best_bucket((1, 1, img_h, img_w))
diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/krea2.py b/python/sglang/multimodal_gen/configs/pipeline_configs/krea2.py
index 104ce82455..e726974f29 100644
--- a/python/sglang/multimodal_gen/configs/pipeline_configs/krea2.py
+++ b/python/sglang/multimodal_gen/configs/pipeline_configs/krea2.py
@@ -6,6 +6,9 @@ from diffusers.image_processor import VaeImageProcessor
from sglang.multimodal_gen.configs.models import DiTConfig, VAEConfig
from sglang.multimodal_gen.configs.models.dits.krea2 import Krea2DitConfig
+from sglang.multimodal_gen.configs.models.vaes.base import (
+ get_channelwise_decode_scale_and_shift,
+)
from sglang.multimodal_gen.configs.models.vaes.qwenimage import QwenImageVAEConfig
from sglang.multimodal_gen.configs.pipeline_configs.base import (
ImagePipelineConfig,
@@ -146,16 +149,9 @@ class Krea2PipelineConfig(ImagePipelineConfig):
# --- VAE decode (same as Qwen-Image: latents * std + mean) ---
def get_decode_scale_and_shift(self, device, dtype, vae):
- vae_arch = self.vae_config.arch_config
- scaling_factor = 1.0 / torch.tensor(vae_arch.latents_std, device=device).view(
- 1, vae_arch.z_dim, 1, 1, 1
- ).to(device, dtype)
- shift_factor = (
- torch.tensor(vae_arch.latents_mean)
- .view(1, vae_arch.z_dim, 1, 1, 1)
- .to(device, dtype)
+ return get_channelwise_decode_scale_and_shift(
+ self.vae_config.arch_config, device, dtype
)
- return scaling_factor, shift_factor
def post_denoising_loop(self, latents, batch):
latents, batch_size, channels, height, width = self._unpad_and_unpack_latents(
diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/longcat_image.py b/python/sglang/multimodal_gen/configs/pipeline_configs/longcat_image.py
index 822e2a6374..f0d480c487 100644
--- a/python/sglang/multimodal_gen/configs/pipeline_configs/longcat_image.py
+++ b/python/sglang/multimodal_gen/configs/pipeline_configs/longcat_image.py
@@ -17,6 +17,7 @@ from sglang.multimodal_gen.configs.pipeline_configs.base import (
ImagePipelineConfig,
ModelTaskType,
TextConditioningOutput,
+ pack_latents_2x2,
)
from sglang.multimodal_gen.configs.pipeline_configs.model_deployment_config import (
ModelDeploymentConfig,
@@ -171,17 +172,6 @@ def _unpack_latents(latents, height, width, vae_scale_factor):
return latents
-def _pack_latents(latents, batch_size, num_channels_latents, height, width):
- latents = latents.view(
- batch_size, num_channels_latents, height // 2, 2, width // 2, 2
- )
- latents = latents.permute(0, 2, 4, 1, 3, 5)
- latents = latents.reshape(
- batch_size, (height // 2) * (width // 2), num_channels_latents * 4
- )
- return latents
-
-
def _calculate_shift(
image_seq_len,
base_seq_len: int = 256,
@@ -293,7 +283,7 @@ class LongCatImagePipelineConfig(ImagePipelineConfig):
def maybe_pack_latents(self, latents, batch_size, batch):
num_channels_latents = self.dit_config.arch_config.num_channels_latents
_, _, h, w = latents.shape
- return _pack_latents(latents, batch_size, num_channels_latents, h, w)
+ return pack_latents_2x2(latents, batch_size, num_channels_latents, h, w)
def maybe_prepare_latent_ids(self, latents):
# latents shape after packing: [B, (h//2)*(w//2), C*4]
@@ -517,7 +507,7 @@ class LongCatImageEditPipelineConfig(LongCatImagePipelineConfig):
batch_size // latent_condition.shape[0], 1, 1, 1
)
_, num_channels_latents, height, width = latent_condition.shape
- return _pack_latents(
+ return pack_latents_2x2(
latent_condition, batch_size, num_channels_latents, height, width
)
diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/ltx_2.py b/python/sglang/multimodal_gen/configs/pipeline_configs/ltx_2.py
index b127d8c8e4..872dc86909 100644
--- a/python/sglang/multimodal_gen/configs/pipeline_configs/ltx_2.py
+++ b/python/sglang/multimodal_gen/configs/pipeline_configs/ltx_2.py
@@ -11,7 +11,10 @@ from sglang.multimodal_gen.configs.models.encoders import (
)
from sglang.multimodal_gen.configs.models.encoders.gemma_3 import Gemma3Config
from sglang.multimodal_gen.configs.models.vaes.ltx_audio import LTXAudioVAEConfig
-from sglang.multimodal_gen.configs.models.vaes.ltx_video import LTXVideoVAEConfig
+from sglang.multimodal_gen.configs.models.vaes.ltx_video import (
+ LTXVideoVAEConfig,
+ get_ltx_video_decode_scale_and_shift,
+)
from sglang.multimodal_gen.configs.pipeline_configs.base import (
ModelTaskType,
PipelineConfig,
@@ -564,36 +567,9 @@ class LTX2PipelineConfig(PipelineConfig):
)
def get_decode_scale_and_shift(self, device, dtype, vae):
- latents_mean = getattr(vae, "latents_mean", None)
- latents_std = getattr(vae, "latents_std", None)
-
- scaling_factor = (
- getattr(getattr(vae, "config", None), "scaling_factor", None)
- or getattr(vae, "scaling_factor", None)
- or getattr(self.vae_config.arch_config, "scaling_factor", None)
- or 1.0
+ return get_ltx_video_decode_scale_and_shift(
+ device, dtype, vae, self.vae_config.arch_config
)
- if isinstance(scaling_factor, (int, float)) and float(scaling_factor) == 0.0:
- scaling_factor = 1.0
-
- if isinstance(latents_mean, torch.Tensor) and isinstance(
- latents_std, torch.Tensor
- ):
- latents_mean = latents_mean.to(device=device, dtype=dtype).view(
- 1, -1, 1, 1, 1
- )
- latents_std = latents_std.to(device=device, dtype=dtype).view(
- 1, -1, 1, 1, 1
- )
- sf = torch.tensor(float(scaling_factor), device=device, dtype=dtype).view(
- 1, 1, 1, 1, 1
- )
- return sf / latents_std, latents_mean
-
- sf = torch.tensor(float(scaling_factor), device=device, dtype=dtype).view(
- 1, 1, 1, 1, 1
- )
- return sf, None
@staticmethod
def _unpack_latents(
@@ -626,29 +602,6 @@ class LTX2PipelineConfig(PipelineConfig):
)
return latents
- @staticmethod
- def _denormalize_latents(
- latents: torch.Tensor,
- latents_mean: torch.Tensor,
- latents_std: torch.Tensor,
- scaling_factor: float = 1.0,
- ) -> torch.Tensor:
- # Denormalize latents across the channel dimension [B, C, F, H, W]
- latents_mean = latents_mean.view(1, -1, 1, 1, 1).to(
- latents.device, latents.dtype
- )
- latents_std = latents_std.view(1, -1, 1, 1, 1).to(latents.device, latents.dtype)
- latents = latents * latents_std / scaling_factor + latents_mean
- return latents
-
- @staticmethod
- def _denormalize_audio_latents(
- latents: torch.Tensor, latents_mean: torch.Tensor, latents_std: torch.Tensor
- ):
- latents_mean = latents_mean.to(latents.device, latents.dtype)
- latents_std = latents_std.to(latents.device, latents.dtype)
- return (latents * latents_std) + latents_mean
-
@staticmethod
def _unpack_audio_latents(
latents: torch.Tensor,
diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/minimax_h3.py b/python/sglang/multimodal_gen/configs/pipeline_configs/minimax_h3.py
index c99edc832c..e4da4079a3 100644
--- a/python/sglang/multimodal_gen/configs/pipeline_configs/minimax_h3.py
+++ b/python/sglang/multimodal_gen/configs/pipeline_configs/minimax_h3.py
@@ -70,16 +70,9 @@ class MiniMaxH3PipelineConfig(PipelineConfig):
output_audio_channels: int | None = 2
output_av_drift_tolerance_s: float | None = 0.25
- def accepts_audio_input(self) -> bool:
- return True
-
def supports_disaggregation(self) -> bool:
return False
- @property
- def requires_audio_output(self) -> bool:
- return True
-
def get_model_deployment_config(self) -> ModelDeploymentConfig:
return ModelDeploymentConfig(
speed_mode_enable_torch_compile_by_default=False,
diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/mova.py b/python/sglang/multimodal_gen/configs/pipeline_configs/mova.py
index 7e09301bc7..9371a778eb 100644
--- a/python/sglang/multimodal_gen/configs/pipeline_configs/mova.py
+++ b/python/sglang/multimodal_gen/configs/pipeline_configs/mova.py
@@ -154,19 +154,6 @@ class MOVAPipelineConfig(PipelineConfig):
latent_T = (num_samples + audio_vae.hop_length - 1) // audio_vae.hop_length
return (batch_size, audio_vae.latent_dim, latent_T)
- def normalize_video_latents(self, latents: torch.Tensor, video_vae) -> torch.Tensor:
- latents_mean = getattr(video_vae.config, "latents_mean", None)
- latents_std = getattr(video_vae.config, "latents_std", None)
- if latents_mean is None or latents_std is None:
- return latents
- mean = torch.tensor(
- latents_mean, device=latents.device, dtype=latents.dtype
- ).view(1, video_vae.config.z_dim, 1, 1, 1)
- inv_std = (
- 1.0 / torch.tensor(latents_std, device=latents.device, dtype=latents.dtype)
- ).view(1, video_vae.config.z_dim, 1, 1, 1)
- return (latents - mean) * inv_std
-
def denormalize_video_latents(
self, latents: torch.Tensor, video_vae
) -> torch.Tensor:
diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/qwen_image.py b/python/sglang/multimodal_gen/configs/pipeline_configs/qwen_image.py
index c99622e357..f57314ead6 100644
--- a/python/sglang/multimodal_gen/configs/pipeline_configs/qwen_image.py
+++ b/python/sglang/multimodal_gen/configs/pipeline_configs/qwen_image.py
@@ -12,11 +12,15 @@ from sglang.multimodal_gen.configs.models.dits.qwenimage import (
QwenImageEditPlus_2511_DitConfig,
)
from sglang.multimodal_gen.configs.models.encoders.qwen_image import Qwen2_5VLConfig
+from sglang.multimodal_gen.configs.models.vaes.base import (
+ get_channelwise_decode_scale_and_shift,
+)
from sglang.multimodal_gen.configs.models.vaes.qwenimage import QwenImageVAEConfig
from sglang.multimodal_gen.configs.pipeline_configs.base import (
ImagePipelineConfig,
ModelTaskType,
maybe_unpad_latents,
+ pack_latents_2x2,
pad_text_embeddings_with_mask,
shard_rotary_emb_for_sp,
)
@@ -141,19 +145,6 @@ def _shard_qwen_edit_freqs_cis_for_sp(freqs_cis, noisy_img_seq_len, device):
)
-# Copied from diffusers.pipelines.qwenimage.pipeline_qwenimage.QwenImagePipeline._pack_latents
-def _pack_latents(latents, batch_size, num_channels_latents, height, width):
- latents = latents.view(
- batch_size, num_channels_latents, height // 2, 2, width // 2, 2
- )
- latents = latents.permute(0, 2, 4, 1, 3, 5)
- latents = latents.reshape(
- batch_size, (height // 2) * (width // 2), num_channels_latents * 4
- )
-
- return latents
-
-
@dataclass
class QwenImagePipelineConfig(QwenImageRolloutPipelineMixin, ImagePipelineConfig):
"""Configuration for the QwenImage pipeline."""
@@ -292,19 +283,14 @@ class QwenImagePipelineConfig(QwenImageRolloutPipelineMixin, ImagePipelineConfig
width = 2 * (batch.width // (vae_scale_factor * 2))
num_channels_latents = self.dit_config.arch_config.in_channels // 4
# pack latents
- return _pack_latents(latents, batch_size, num_channels_latents, height, width)
+ return pack_latents_2x2(
+ latents, batch_size, num_channels_latents, height, width
+ )
def get_decode_scale_and_shift(self, device, dtype, vae):
- vae_arch_config = self.vae_config.arch_config
- scaling_factor = 1.0 / torch.tensor(
- vae_arch_config.latents_std, device=device
- ).view(1, vae_arch_config.z_dim, 1, 1, 1).to(device, dtype)
- shift_factor = (
- torch.tensor(vae_arch_config.latents_mean)
- .view(1, vae_arch_config.z_dim, 1, 1, 1)
- .to(device, dtype)
+ return get_channelwise_decode_scale_and_shift(
+ self.vae_config.arch_config, device, dtype
)
- return scaling_factor, shift_factor
@staticmethod
def get_freqs_cis(img_shapes, txt_seq_lens, rotary_emb, device, dtype):
@@ -594,7 +580,7 @@ class QwenImageEditPipelineConfig(QwenImagePipelineConfig):
image_latents = latent_condition
image_latent_height, image_latent_width = image_latents.shape[3:]
num_channels_latents = self.dit_config.arch_config.in_channels // 4
- image_latents = _pack_latents(
+ image_latents = pack_latents_2x2(
image_latents,
batch_size,
num_channels_latents,
@@ -639,21 +625,6 @@ VAE_IMAGE_SIZE = 1024 * 1024
class QwenImageEditPlusPipelineConfig(QwenImageEditPipelineConfig):
task_type: ModelTaskType = ModelTaskType.I2I
- def _get_condition_image_sizes(self, batch) -> list[tuple[int, int]]:
- image = batch.condition_image
- if not isinstance(image, list):
- image = [image]
-
- condition_image_sizes = []
- for img in image:
- image_width, image_height = img.size
- edit_width, edit_height, _ = _calculate_dimensions(
- VAE_IMAGE_SIZE, image_width / image_height
- )
- condition_image_sizes.append((edit_width, edit_height))
-
- return condition_image_sizes
-
def prepare_image_processor_kwargs(self, batch, neg=False) -> dict:
prompt = batch.prompt if not neg else batch.negative_prompt
if not prompt:
@@ -686,14 +657,6 @@ class QwenImageEditPlusPipelineConfig(QwenImageEditPipelineConfig):
def prepare_calculated_size(self, image):
return self.calculate_vae_image_size(image, image.width, image.height)
- def resize_condition_image(self, images, target_width, target_height):
- if not isinstance(images, list):
- images = [images]
- new_images = []
- for img, width, height in zip(images, target_width, target_height):
- new_images.append(resize(img, height, width, resize_mode="default"))
- return new_images
-
def calculate_condition_image_size(self, image, width, height) -> tuple[int, int]:
calculated_width, calculated_height, _ = _calculate_dimensions(
CONDITION_IMAGE_SIZE, width / height
diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/sana_wm.py b/python/sglang/multimodal_gen/configs/pipeline_configs/sana_wm.py
index 3dc674f6c7..ac32a230f4 100644
--- a/python/sglang/multimodal_gen/configs/pipeline_configs/sana_wm.py
+++ b/python/sglang/multimodal_gen/configs/pipeline_configs/sana_wm.py
@@ -11,7 +11,10 @@ from sglang.multimodal_gen.configs.models.dits.sana_wm import SanaWMConfig
from sglang.multimodal_gen.configs.models.encoders import BaseEncoderOutput
from sglang.multimodal_gen.configs.models.encoders.base import EncoderConfig
from sglang.multimodal_gen.configs.models.encoders.gemma2 import Gemma2Config
-from sglang.multimodal_gen.configs.models.vaes.ltx_video import LTXVideoVAEConfig
+from sglang.multimodal_gen.configs.models.vaes.ltx_video import (
+ LTXVideoVAEConfig,
+ get_ltx_video_decode_scale_and_shift,
+)
from sglang.multimodal_gen.configs.pipeline_configs.base import (
ModelTaskType,
PipelineConfig,
@@ -286,42 +289,9 @@ class SanaWMPipelineConfig(PipelineConfig):
return latents
def get_decode_scale_and_shift(self, device, dtype, vae):
- """Invert the LTX-2 latent normalization used before denoising.
-
- SANA-WM uses the LTX-2 VAE. Upstream encodes as
- ``(z - latents_mean) * scaling_factor / latents_std`` and decodes by
- applying the inverse transform.
- """
- latents_mean = getattr(vae, "latents_mean", None)
- latents_std = getattr(vae, "latents_std", None)
-
- scaling_factor = (
- getattr(getattr(vae, "config", None), "scaling_factor", None)
- or getattr(vae, "scaling_factor", None)
- or getattr(self.vae_config.arch_config, "scaling_factor", None)
- or 1.0
+ return get_ltx_video_decode_scale_and_shift(
+ device, dtype, vae, self.vae_config.arch_config
)
- if isinstance(scaling_factor, (int, float)) and float(scaling_factor) == 0.0:
- scaling_factor = 1.0
-
- if isinstance(latents_mean, torch.Tensor) and isinstance(
- latents_std, torch.Tensor
- ):
- latents_mean = latents_mean.to(device=device, dtype=dtype).view(
- 1, -1, 1, 1, 1
- )
- latents_std = latents_std.to(device=device, dtype=dtype).view(
- 1, -1, 1, 1, 1
- )
- sf = torch.tensor(float(scaling_factor), device=device, dtype=dtype).view(
- 1, 1, 1, 1, 1
- )
- return sf / latents_std, latents_mean
-
- sf = torch.tensor(float(scaling_factor), device=device, dtype=dtype).view(
- 1, 1, 1, 1, 1
- )
- return sf, None
class SanaWMRealtimeConfig(SanaWMPipelineConfig):
diff --git a/python/sglang/multimodal_gen/configs/sample/action.py b/python/sglang/multimodal_gen/configs/sample/action.py
index c3f341516e..658fc2b356 100644
--- a/python/sglang/multimodal_gen/configs/sample/action.py
+++ b/python/sglang/multimodal_gen/configs/sample/action.py
@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Any
from sglang.multimodal_gen.configs.sample.sampling_params import (
DataType,
+ SamplingParams,
_sanitize_filename,
)
from sglang.multimodal_gen.configs.task_type import ModelTaskType
@@ -65,15 +66,7 @@ class ActionSamplingParams:
if env_steps is not None and self.num_inference_steps is not None:
self.num_inference_steps = int(env_steps)
- def build_request_extra(self) -> dict[str, Any]:
- extra = {}
- diffusers_kwargs = getattr(self, "diffusers_kwargs", None)
- if diffusers_kwargs:
- extra["diffusers_kwargs"] = diffusers_kwargs
- explicit_fields = getattr(self, "_explicit_fields", None)
- if explicit_fields is not None:
- extra["explicit_fields"] = sorted(explicit_fields)
- return extra
+ build_request_extra = SamplingParams.build_request_extra
def apply_request_extra(self, req: Any) -> None:
req.extra.update(self.build_request_extra())
diff --git a/python/sglang/multimodal_gen/configs/sample/minimax_h3.py b/python/sglang/multimodal_gen/configs/sample/minimax_h3.py
index 6b18ebfcf7..f679e836f0 100644
--- a/python/sglang/multimodal_gen/configs/sample/minimax_h3.py
+++ b/python/sglang/multimodal_gen/configs/sample/minimax_h3.py
@@ -1,5 +1,4 @@
# SPDX-License-Identifier: Apache-2.0
-import math
import os
from collections.abc import Mapping
from dataclasses import dataclass, field
@@ -8,6 +7,7 @@ from typing import Any, ClassVar
import msgspec
from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
+from sglang.multimodal_gen.configs.utils import optional_positive_finite_float
_MINIMAX_H3_MAX_SIGNED_SEED = (1 << 63) - 1
@@ -23,17 +23,6 @@ def _optional_unit_float(value: Any, field_name: str) -> float | None:
return out
-def _optional_positive_finite_float(value: Any, field_name: str) -> float | None:
- if value is None:
- return None
- if isinstance(value, bool) or not isinstance(value, (int, float)):
- raise ValueError(f"{field_name} must be a number")
- out = float(value)
- if not math.isfinite(out) or out <= 0.0:
- raise ValueError(f"{field_name} must be a positive finite number")
- return out
-
-
@dataclass
class MiniMaxH3SamplingParams(SamplingParams):
# The video/audio sigma schedules include both interval endpoints.
@@ -226,8 +215,8 @@ class MiniMaxH3SamplingParams(SamplingParams):
if field_name in self.target
}
super()._validate()
- _optional_positive_finite_float(self.flow_shift, "flow_shift")
- _optional_positive_finite_float(self.audio_flow_shift, "audio_flow_shift")
+ optional_positive_finite_float(self.flow_shift, "flow_shift")
+ optional_positive_finite_float(self.audio_flow_shift, "audio_flow_shift")
if self.enable_frame_interpolation:
raise ValueError(
"MiniMax H3 does not support enable_frame_interpolation: the "
diff --git a/python/sglang/multimodal_gen/configs/sample/sampling_params.py b/python/sglang/multimodal_gen/configs/sample/sampling_params.py
index 4159cf624f..51707c0477 100644
--- a/python/sglang/multimodal_gen/configs/sample/sampling_params.py
+++ b/python/sglang/multimodal_gen/configs/sample/sampling_params.py
@@ -758,10 +758,6 @@ class SamplingParams:
RLRolloutArgs.validate_sampling_params(self)
- def check_sampling_param(self):
- # Keep backward-compatibility for old call sites.
- self._validate()
-
def _validate_with_pipeline_config(self, pipeline_config):
"""
check if the sampling params is compatible and valid with server_args
diff --git a/python/sglang/multimodal_gen/configs/utils.py b/python/sglang/multimodal_gen/configs/utils.py
index 4388dccb7c..1084c6a6bb 100644
--- a/python/sglang/multimodal_gen/configs/utils.py
+++ b/python/sglang/multimodal_gen/configs/utils.py
@@ -1,12 +1,23 @@
# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo
-import argparse
+import math
import os
from dataclasses import fields
from operator import attrgetter
from typing import Any
+def optional_positive_finite_float(value: Any, field_name: str) -> float | None:
+ if value is None:
+ return None
+ if isinstance(value, bool) or not isinstance(value, (int, float)):
+ raise ValueError(f"{field_name} must be a number")
+ out = float(value)
+ if not math.isfinite(out) or out <= 0.0:
+ raise ValueError(f"{field_name} must be a positive finite number")
+ return out
+
+
def expand_path_fields(obj) -> None:
"""Expand paths in dataclass configuration without modifying other fields."""
for f in fields(obj):
@@ -81,15 +92,3 @@ def update_config_from_args(
args_dict.pop(key)
return len(args_to_remove) > 0
-
-
-def clean_cli_args(args: argparse.Namespace) -> dict[str, Any]:
- """
- Clean the arguments by removing the ones that not explicitly provided by the user.
- """
- provided_args = {}
- for k, v in vars(args).items():
- if v is not None and hasattr(args, "_provided") and k in args._provided:
- provided_args[k] = v
-
- return provided_args
diff --git a/python/sglang/multimodal_gen/envs.py b/python/sglang/multimodal_gen/envs.py
index 4f0c8cfd0b..4d20d27861 100644
--- a/python/sglang/multimodal_gen/envs.py
+++ b/python/sglang/multimodal_gen/envs.py
@@ -5,12 +5,21 @@
import logging
import os
+import warnings
from typing import TYPE_CHECKING, Any, Callable
from sglang.multimodal_gen.runtime.utils.common import get_bool_env_var
logger = logging.getLogger(__name__)
+if "SGLANG_DIFFUSION_TRACE_FUNCTION" in os.environ:
+ warnings.warn(
+ "SGLANG_DIFFUSION_TRACE_FUNCTION was unused and has been removed. "
+ "Use Python profiling tools for function-level tracing.",
+ FutureWarning,
+ stacklevel=2,
+ )
+
if TYPE_CHECKING:
SGLANG_DIFFUSION_NCCL_SO_PATH: str | None = None
LD_LIBRARY_PATH: str | None = None
@@ -20,7 +29,6 @@ if TYPE_CHECKING:
SGLANG_DIFFUSION_CONFIG_ROOT: str = os.path.expanduser("~/.config/sgl_diffusion")
SGLANG_DIFFUSION_LOGGING_LEVEL: str = "INFO"
SGLANG_DIFFUSION_LOGGING_PREFIX: str = ""
- SGLANG_DIFFUSION_TRACE_FUNCTION: int = 0
SGLANG_DIFFUSION_DISABLE_EARLY_VAE_DECODER_CAST: bool = False
SGLANG_DIFFUSION_DISABLE_VAE_DECODER_STORE: bool = False
SGLANG_DIFFUSION_DISABLE_MAPPED_WILLNEED: bool = False
@@ -229,10 +237,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
),
# if set, SGLANG_DIFFUSION_LOGGING_PREFIX will be prepended to all log messages
"SGLANG_DIFFUSION_LOGGING_PREFIX": _lazy_str("SGLANG_DIFFUSION_LOGGING_PREFIX", ""),
- # Trace function calls
- # If set to 1, sgl_diffusion will trace function calls
- # Useful for debugging
- "SGLANG_DIFFUSION_TRACE_FUNCTION": _lazy_int("SGLANG_DIFFUSION_TRACE_FUNCTION", 0),
# Path to the attention configuration file. Only used for sliding tile
# attention for now.
"SGLANG_DIFFUSION_ATTENTION_CONFIG": _lazy_path(
diff --git a/python/sglang/multimodal_gen/runtime/breakable_cuda_graph/model_padders/ideogram.py b/python/sglang/multimodal_gen/runtime/breakable_cuda_graph/model_padders/ideogram.py
index d4f7ec936a..104ba66161 100644
--- a/python/sglang/multimodal_gen/runtime/breakable_cuda_graph/model_padders/ideogram.py
+++ b/python/sglang/multimodal_gen/runtime/breakable_cuda_graph/model_padders/ideogram.py
@@ -30,16 +30,8 @@ def is_ideogram_transformer(current_model: Any, call_kwargs: dict) -> bool:
)
-def _unwrap_model(current_model: Any) -> Any:
- for attr in ("module", "_orig_mod"):
- wrapped = getattr(current_model, attr, None)
- if wrapped is not None:
- current_model = wrapped
- return current_model
-
-
def _dynamic_mask_meta(current_model: Any) -> DynamicVarlenMaskMeta:
- model = _unwrap_model(current_model)
+ model = bcg_utils.unwrap_model(current_model)
meta = getattr(model, _DYNAMIC_MASK_META_ATTR, None)
if not isinstance(meta, DynamicVarlenMaskMeta):
meta = DynamicVarlenMaskMeta()
diff --git a/python/sglang/multimodal_gen/runtime/breakable_cuda_graph/model_padders/zimage.py b/python/sglang/multimodal_gen/runtime/breakable_cuda_graph/model_padders/zimage.py
index 7517ee3277..7e6134c47f 100644
--- a/python/sglang/multimodal_gen/runtime/breakable_cuda_graph/model_padders/zimage.py
+++ b/python/sglang/multimodal_gen/runtime/breakable_cuda_graph/model_padders/zimage.py
@@ -55,16 +55,8 @@ def _pad_caption(obj: Any, *, target: int) -> Any:
return obj
-def _unwrap_model(current_model: Any) -> Any:
- for attr in ("module", "_orig_mod"):
- wrapped = getattr(current_model, attr, None)
- if wrapped is not None:
- current_model = wrapped
- return current_model
-
-
def _build_caption_freqs(current_model: Any, *, target: int, device: torch.device):
- rotary_emb = getattr(_unwrap_model(current_model), "rotary_emb", None)
+ rotary_emb = getattr(bcg_utils.unwrap_model(current_model), "rotary_emb", None)
if rotary_emb is None:
return None
diff --git a/python/sglang/multimodal_gen/runtime/breakable_cuda_graph/prompt_padding.py b/python/sglang/multimodal_gen/runtime/breakable_cuda_graph/prompt_padding.py
index e856bcc0c5..b9721d41dd 100644
--- a/python/sglang/multimodal_gen/runtime/breakable_cuda_graph/prompt_padding.py
+++ b/python/sglang/multimodal_gen/runtime/breakable_cuda_graph/prompt_padding.py
@@ -249,6 +249,14 @@ def pad_masked_prompt_kwargs(call_kwargs: dict, buckets: tuple[int, ...]) -> dic
return out
+def unwrap_model(current_model: Any) -> Any:
+ for attr in ("module", "_orig_mod"):
+ wrapped = getattr(current_model, attr, None)
+ if wrapped is not None:
+ current_model = wrapped
+ return current_model
+
+
def transformer_class_name_matches(current_model: Any, needle: str) -> bool:
"""True when ``current_model`` (or its ``module`` / ``_orig_mod`` wrapper)
is a transformer whose qualified class name contains ``needle``."""
diff --git a/python/sglang/multimodal_gen/runtime/disaggregation/disagg_args.py b/python/sglang/multimodal_gen/runtime/disaggregation/disagg_args.py
deleted file mode 100644
index 5e6c35922a..0000000000
--- a/python/sglang/multimodal_gen/runtime/disaggregation/disagg_args.py
+++ /dev/null
@@ -1,28 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-"""Compatibility shim for disaggregated diffusion argument helpers."""
-
-from __future__ import annotations
-
-import argparse
-
-from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
-from sglang.multimodal_gen.runtime.server_args.disagg import DisaggServerArgsMixin
-
-# Keep the historical disagg_args import path working.
-DISAGG_RESULT_PORT_OFFSETS = DisaggServerArgsMixin.DISAGG_RESULT_PORT_OFFSETS
-DisaggArgsMixin = DisaggServerArgsMixin
-
-
-def add_disagg_cli_args(parser: argparse.ArgumentParser) -> None:
- """Register disaggregated-diffusion CLI args through ServerArgs."""
-
- from sglang.multimodal_gen.runtime.server_args import ServerArgs
-
- ServerArgs.add_disagg_cli_args(parser)
-
-
-def convert_disagg_role_string(kwargs: dict) -> None:
- """Convert ``disagg_role`` from string to ``RoleType`` enum in-place."""
-
- if "disagg_role" in kwargs and isinstance(kwargs["disagg_role"], str):
- kwargs["disagg_role"] = RoleType.from_string(kwargs["disagg_role"])
diff --git a/python/sglang/multimodal_gen/runtime/disaggregation/dispatch_policy.py b/python/sglang/multimodal_gen/runtime/disaggregation/dispatch_policy.py
index 1382a5ef36..f39c97e3d0 100644
--- a/python/sglang/multimodal_gen/runtime/disaggregation/dispatch_policy.py
+++ b/python/sglang/multimodal_gen/runtime/disaggregation/dispatch_policy.py
@@ -2,11 +2,8 @@
"""Dispatch policies for multi-instance disaggregated diffusion pipelines."""
import abc
-import logging
import threading
-logger = logging.getLogger(__name__)
-
class DispatchPolicy(abc.ABC):
def __init__(self, num_instances: int):
@@ -19,16 +16,9 @@ class DispatchPolicy(abc.ABC):
return self._num_instances
@abc.abstractmethod
- def select(self, active_counts: list[int] | None = None) -> int: ...
-
def select_with_capacity(self, free_slots: list[int]) -> int | None:
"""Select an instance that has free capacity, or None if all full."""
- if not any(s > 0 for s in free_slots):
- return None
- return self.select(active_counts=None)
-
- def record_completion(self, instance_id: int) -> None:
- pass
+ ...
class RoundRobin(DispatchPolicy):
@@ -37,12 +27,6 @@ class RoundRobin(DispatchPolicy):
self._lock = threading.Lock()
self._next = 0
- def select(self, active_counts: list[int] | None = None) -> int:
- with self._lock:
- chosen = self._next
- self._next = (self._next + 1) % self._num_instances
- return chosen
-
def select_with_capacity(self, free_slots: list[int]) -> int | None:
with self._lock:
for _ in range(self._num_instances):
@@ -56,43 +40,11 @@ class RoundRobin(DispatchPolicy):
class MaxFreeSlotsFirst(DispatchPolicy):
"""Dispatch to the instance with the most free slots."""
- def __init__(self, num_instances: int, max_slots_per_instance: int = 1):
+ def __init__(self, num_instances: int):
super().__init__(num_instances)
- self._max_slots = max_slots_per_instance
self._lock = threading.Lock()
self._tiebreak = 0
- def select(self, active_counts: list[int] | None = None) -> int:
- with self._lock:
- if active_counts is None or len(active_counts) != self._num_instances:
- chosen = self._tiebreak % self._num_instances
- self._tiebreak += 1
- return chosen
-
- best_id = 0
- best_free = self._max_slots - active_counts[0]
- for i in range(1, self._num_instances):
- free = self._max_slots - active_counts[i]
- if free > best_free:
- best_free = free
- best_id = i
- elif free == best_free:
- if i == (self._tiebreak % self._num_instances):
- best_id = i
-
- self._tiebreak += 1
-
- if best_free <= 0:
- logger.warning(
- "All %d instances are at capacity (%d slots each), "
- "dispatching to instance %d anyway",
- self._num_instances,
- self._max_slots,
- best_id,
- )
-
- return best_id
-
def select_with_capacity(self, free_slots: list[int]) -> int | None:
with self._lock:
best_id = -1
@@ -121,26 +73,10 @@ class PoolDispatcher:
num_denoisers: int,
num_decoders: int,
policy_name: str = "round_robin",
- **kwargs,
):
- self.encoder_policy = create_dispatch_policy(
- policy_name, num_encoders, **kwargs
- )
- self.denoiser_policy = create_dispatch_policy(
- policy_name, num_denoisers, **kwargs
- )
- self.decoder_policy = create_dispatch_policy(
- policy_name, num_decoders, **kwargs
- )
-
- def select_encoder(self, active_counts: list[int] | None = None) -> int:
- return self.encoder_policy.select(active_counts)
-
- def select_denoiser(self, active_counts: list[int] | None = None) -> int:
- return self.denoiser_policy.select(active_counts)
-
- def select_decoder(self, active_counts: list[int] | None = None) -> int:
- return self.decoder_policy.select(active_counts)
+ self.encoder_policy = create_dispatch_policy(policy_name, num_encoders)
+ self.denoiser_policy = create_dispatch_policy(policy_name, num_denoisers)
+ self.decoder_policy = create_dispatch_policy(policy_name, num_decoders)
def select_encoder_with_capacity(self, free_slots: list[int]) -> int | None:
return self.encoder_policy.select_with_capacity(free_slots)
@@ -152,7 +88,7 @@ class PoolDispatcher:
return self.decoder_policy.select_with_capacity(free_slots)
-def create_dispatch_policy(name: str, num_instances: int, **kwargs) -> DispatchPolicy:
+def create_dispatch_policy(name: str, num_instances: int) -> DispatchPolicy:
policies = {
"round_robin": RoundRobin,
"max_free_slots": MaxFreeSlotsFirst,
@@ -162,4 +98,4 @@ def create_dispatch_policy(name: str, num_instances: int, **kwargs) -> DispatchP
raise ValueError(
f"Unknown dispatch policy '{name}'. Available: {list(policies.keys())}"
)
- return cls(num_instances=num_instances, **kwargs)
+ return cls(num_instances=num_instances)
diff --git a/python/sglang/multimodal_gen/runtime/disaggregation/metrics.py b/python/sglang/multimodal_gen/runtime/disaggregation/metrics.py
index cfe0b9c706..b65d506a03 100644
--- a/python/sglang/multimodal_gen/runtime/disaggregation/metrics.py
+++ b/python/sglang/multimodal_gen/runtime/disaggregation/metrics.py
@@ -6,12 +6,6 @@ import time
from dataclasses import dataclass
-@dataclass
-class _RequestTiming:
- start_time: float
- stage_start: float = 0.0
-
-
@dataclass
class RoleStats:
role: str
@@ -54,7 +48,7 @@ class DisaggMetrics:
self._failed = 0
self._timed_out = 0
- self._in_flight: dict[str, _RequestTiming] = {}
+ self._in_flight: dict[str, float] = {}
self._last_latency = 0.0
self._max_latency = 0.0
@@ -71,14 +65,14 @@ class DisaggMetrics:
def record_request_start(self, request_id: str) -> None:
with self._lock:
- self._in_flight[request_id] = _RequestTiming(start_time=time.monotonic())
+ self._in_flight[request_id] = time.monotonic()
def record_request_complete(self, request_id: str) -> None:
now = time.monotonic()
with self._lock:
- timing = self._in_flight.pop(request_id, None)
- if timing is not None:
- latency = now - timing.start_time
+ start_time = self._in_flight.pop(request_id, None)
+ if start_time is not None:
+ latency = now - start_time
self._last_latency = latency
self._max_latency = max(self._max_latency, latency)
self._total_latency += latency
diff --git a/python/sglang/multimodal_gen/runtime/disaggregation/orchestrator.py b/python/sglang/multimodal_gen/runtime/disaggregation/orchestrator.py
index fd3f85157e..3be3052097 100644
--- a/python/sglang/multimodal_gen/runtime/disaggregation/orchestrator.py
+++ b/python/sglang/multimodal_gen/runtime/disaggregation/orchestrator.py
@@ -865,8 +865,8 @@ class DiffusionServer:
if idx is None:
break
entry = self._denoiser_tta.popleft()
- self._transfer_dispatch_to_denoiser(
- entry.request_id, entry.transfer_state, idx
+ self._transfer_dispatch_to_receiver(
+ entry.request_id, entry.transfer_state, idx, RoleType.DENOISER
)
def _drain_decoder_tta(self) -> None:
@@ -877,8 +877,8 @@ class DiffusionServer:
if idx is None:
break
entry = self._decoder_tta.popleft()
- self._transfer_dispatch_to_decoder(
- entry.request_id, entry.transfer_state, idx
+ self._transfer_dispatch_to_receiver(
+ entry.request_id, entry.transfer_state, idx, RoleType.DECODER
)
def _extract_request_id(self, frames: list) -> str | None:
@@ -1161,36 +1161,57 @@ class DiffusionServer:
)
receiver_pushes[receiver_idx].send_multipart(encode_transfer_msg(alloc_msg))
- def _transfer_dispatch_to_denoiser(
- self, request_id: str, p2p: _TransferRequestState, denoiser_idx: int
+ def _transfer_dispatch_to_receiver(
+ self,
+ request_id: str,
+ p2p: _TransferRequestState,
+ receiver_idx: int,
+ role: RoleType,
) -> None:
- self._denoiser_free_slots[denoiser_idx] -= 1
- p2p.receiver_instance = denoiser_idx
+ if role == RoleType.DENOISER:
+ free_slots = self._denoiser_free_slots
+ peers = self._denoiser_peers
+ sender_pushes = self._encoder_pushes
+ receiver_pushes = self._denoiser_pushes
+ source_role = "encoder"
+ running_state = RequestState.DENOISING_RUNNING
+ elif role == RoleType.DECODER:
+ free_slots = self._decoder_free_slots
+ peers = self._decoder_peers
+ sender_pushes = self._denoiser_pushes
+ receiver_pushes = self._decoder_pushes
+ source_role = "denoiser"
+ running_state = RequestState.DECODER_RUNNING
+ else:
+ raise ValueError(f"Invalid transfer receiver role: {role}")
+
+ free_slots[receiver_idx] -= 1
+ p2p.receiver_instance = receiver_idx
try:
self._tracker.transition(
request_id,
- RequestState.DENOISING_RUNNING,
- denoiser_instance=denoiser_idx,
+ running_state,
+ **{f"{role.value}_instance": receiver_idx},
)
except ValueError:
pass
- peer_info = self._denoiser_peers.get(denoiser_idx, {})
+ peer_info = peers.get(receiver_idx, {})
if not self._try_fast_path_push(
request_id=request_id,
p2p=p2p,
receiver_peer_info=peer_info,
- sender_pushes=self._encoder_pushes,
- receiver_role_label="denoiser",
- receiver_idx=denoiser_idx,
+ sender_pushes=sender_pushes,
+ receiver_role_label=role.value,
+ receiver_idx=receiver_idx,
):
self._send_slow_path_alloc(
request_id=request_id,
p2p=p2p,
- receiver_pushes=self._denoiser_pushes,
- receiver_idx=denoiser_idx,
- source_role="encoder",
+ receiver_pushes=receiver_pushes,
+ receiver_idx=receiver_idx,
+ source_role=source_role,
)
def _handle_transfer_allocated(self, msg: dict) -> None:
@@ -1378,38 +1399,6 @@ class DiffusionServer:
self._transfer_state.pop(request_id, None)
- def _transfer_dispatch_to_decoder(
- self, request_id: str, p2p: _TransferRequestState, decoder_idx: int
- ) -> None:
- self._decoder_free_slots[decoder_idx] -= 1
- p2p.receiver_instance = decoder_idx
-
- try:
- self._tracker.transition(
- request_id,
- RequestState.DECODER_RUNNING,
- decoder_instance=decoder_idx,
- )
- except ValueError:
- pass
-
- peer_info = self._decoder_peers.get(decoder_idx, {})
- if not self._try_fast_path_push(
- request_id=request_id,
- p2p=p2p,
- receiver_peer_info=peer_info,
- sender_pushes=self._denoiser_pushes,
- receiver_role_label="decoder",
- receiver_idx=decoder_idx,
- ):
- self._send_slow_path_alloc(
- request_id=request_id,
- p2p=p2p,
- receiver_pushes=self._decoder_pushes,
- receiver_idx=decoder_idx,
- source_role="denoiser",
- )
-
def _transfer_return_to_client_from_msg(self, request_id: str, msg: dict) -> None:
with self._lock:
client_identity = self._pending.pop(request_id, None)
diff --git a/python/sglang/multimodal_gen/runtime/disaggregation/scheduler_mixin.py b/python/sglang/multimodal_gen/runtime/disaggregation/scheduler_mixin.py
index 6a0867d393..834b7c8cd0 100644
--- a/python/sglang/multimodal_gen/runtime/disaggregation/scheduler_mixin.py
+++ b/python/sglang/multimodal_gen/runtime/disaggregation/scheduler_mixin.py
@@ -427,9 +427,7 @@ class SchedulerDisaggMixin:
from sglang.multimodal_gen.runtime.disaggregation.metrics import DisaggMetrics
self._disagg_role = server_args.disagg_role
- self._disagg_timeout_s = float(getattr(server_args, "disagg_timeout", 600))
self._disagg_metrics = None
- self._disagg_mode = getattr(server_args, "disagg_mode", False)
self._pool_work_pull = None
self._pool_result_push = None
self._transfer_manager = None
diff --git a/python/sglang/multimodal_gen/runtime/distributed/communication_op.py b/python/sglang/multimodal_gen/runtime/distributed/communication_op.py
index e336e4cf8c..e9b267e216 100644
--- a/python/sglang/multimodal_gen/runtime/distributed/communication_op.py
+++ b/python/sglang/multimodal_gen/runtime/distributed/communication_op.py
@@ -45,11 +45,6 @@ def sequence_model_parallel_all_gather(
return get_sp_group().all_gather(input_, dim)
-def sequence_model_parallel_all_reduce(input_: torch.Tensor) -> torch.Tensor:
- """All-reduce the input tensor across model parallel group."""
- return get_sp_group().all_reduce(input_)
-
-
def cfg_model_parallel_all_gather(
input_: torch.Tensor, dim: int = -1, separate_tensors: bool = False
) -> torch.Tensor:
diff --git a/python/sglang/multimodal_gen/runtime/distributed/device_communicators/base_device_communicator.py b/python/sglang/multimodal_gen/runtime/distributed/device_communicators/base_device_communicator.py
index c7a7dbdc08..b642071dbf 100644
--- a/python/sglang/multimodal_gen/runtime/distributed/device_communicators/base_device_communicator.py
+++ b/python/sglang/multimodal_gen/runtime/distributed/device_communicators/base_device_communicator.py
@@ -270,7 +270,6 @@ class DeviceCommunicatorBase:
self.world_size = dist.get_world_size(cpu_group)
self.ranks = dist.get_process_group_ranks(cpu_group)
self.global_rank = dist.get_rank()
- self.global_world_size = dist.get_world_size()
self.rank_in_group = dist.get_group_rank(self.cpu_group, self.global_rank)
def all_reduce(
diff --git a/python/sglang/multimodal_gen/runtime/distributed/group_coordinator.py b/python/sglang/multimodal_gen/runtime/distributed/group_coordinator.py
index a6fc8d9613..d8cf0c8a88 100644
--- a/python/sglang/multimodal_gen/runtime/distributed/group_coordinator.py
+++ b/python/sglang/multimodal_gen/runtime/distributed/group_coordinator.py
@@ -169,7 +169,6 @@ class GroupCoordinator:
rank_in_group: int # rank inside the group
cpu_group: ProcessGroup # group for CPU communication
device_group: ProcessGroup # group for device communication
- use_device_communicator: bool # whether to use device communicator
device_communicator: DeviceCommunicatorBase # all_to_all_4D communicator
def __init__(
@@ -207,7 +206,6 @@ class GroupCoordinator:
# TODO: fix it for other platforms
self.device = get_local_torch_device()
- self.use_device_communicator = use_device_communicator
self.device_communicator: DeviceCommunicatorBase = None # type: ignore
if use_device_communicator and self.world_size > 1:
communicator_cls = _resolve_all_to_all_communicator_cls()
@@ -228,10 +226,6 @@ class GroupCoordinator:
# srt owns topology, dtype, contiguity, and size dispatch for custom ar
self._init_srt_custom_allreduce()
- # TODO(will): check if this is needed
- # self.use_custom_op_call = current_platform.is_cuda_alike()
- self.use_custom_op_call = False
-
def _init_srt_custom_allreduce(self) -> None:
custom_allreduce_kwargs = {
"group": self.cpu_group,
diff --git a/python/sglang/multimodal_gen/runtime/distributed/parallel_state.py b/python/sglang/multimodal_gen/runtime/distributed/parallel_state.py
index 7f76a04fce..826ae957fb 100644
--- a/python/sglang/multimodal_gen/runtime/distributed/parallel_state.py
+++ b/python/sglang/multimodal_gen/runtime/distributed/parallel_state.py
@@ -767,11 +767,6 @@ def get_sequence_parallel_world_size() -> int:
return get_sp_world_size()
-def get_sequence_parallel_rank() -> int:
- """Return my rank for the sequence parallel group."""
- return get_sp_parallel_rank()
-
-
def get_ulysses_parallel_world_size() -> int:
return get_sp_group().ulysses_world_size
@@ -803,32 +798,6 @@ def get_ring_ctx() -> tuple[int, int]:
return get_ring_parallel_world_size(), get_ring_parallel_rank()
-# PP
-def get_pp_group() -> GroupCoordinator:
- assert _PP is not None, "pipeline model parallel group is not initialized"
- return _PP
-
-
-def get_pipeline_parallel_world_size() -> int:
- """Return world size for the pipeline model parallel group."""
- return get_pp_group().world_size
-
-
-def get_pipeline_parallel_rank() -> int:
- """Return my rank for the pipeline model parallel group."""
- return get_pp_group().rank_in_group
-
-
-def is_pipeline_first_stage() -> bool:
- """Return True if in the first pipeline model parallel stage, False otherwise."""
- return get_pipeline_parallel_rank() == 0
-
-
-def is_pipeline_last_stage() -> bool:
- """Return True if in the last pipeline model parallel stage, False otherwise."""
- return get_pipeline_parallel_rank() == (get_pipeline_parallel_world_size() - 1)
-
-
# CFG
def get_cfg_group() -> GroupCoordinator:
assert _CFG is not None, (
@@ -852,47 +821,6 @@ def get_data_parallel_world_size() -> int:
return get_dp_world_size()
-def get_data_parallel_rank() -> int:
- """Return my rank for the data parallel group."""
- return get_dp_rank()
-
-
-def is_dp_last_group() -> bool:
- """Return True if in the last data parallel group, False otherwise."""
- return (
- get_sequence_parallel_rank() == (get_sequence_parallel_world_size() - 1)
- and get_classifier_free_guidance_rank()
- == (get_classifier_free_guidance_world_size() - 1)
- and get_pipeline_parallel_rank() == (get_pipeline_parallel_world_size() - 1)
- )
-
-
-def get_dit_world_size() -> int:
- """Return world size for the DiT model (excluding VAE)."""
- return (
- get_data_parallel_world_size()
- * get_classifier_free_guidance_world_size()
- * get_sequence_parallel_world_size()
- * get_pipeline_parallel_world_size()
- * get_tensor_model_parallel_world_size()
- )
-
-
-def get_vae_parallel_group() -> ProcessGroup:
- assert _VAE is not None, "VAE parallel group is not initialized"
- return _VAE
-
-
-def get_vae_parallel_world_size() -> int:
- """Return world size for the VAE parallel group."""
- return torch.distributed.get_world_size(group=get_vae_parallel_group())
-
-
-def get_vae_parallel_rank() -> int:
- """Return my rank for the VAE parallel group."""
- return torch.distributed.get_rank(group=get_vae_parallel_group())
-
-
def get_decode_parallel_group_coordinator() -> GroupCoordinator:
assert _VAE_DECODE is not None, "VAE decode parallel group is not initialized"
return _VAE_DECODE
diff --git a/python/sglang/multimodal_gen/runtime/entrypoints/cli/generate.py b/python/sglang/multimodal_gen/runtime/entrypoints/cli/generate.py
index dedcae21c7..3da67502c1 100644
--- a/python/sglang/multimodal_gen/runtime/entrypoints/cli/generate.py
+++ b/python/sglang/multimodal_gen/runtime/entrypoints/cli/generate.py
@@ -215,16 +215,6 @@ class GenerateSubcommand(CLISubcommand):
def __init__(self) -> None:
self.name = "generate"
super().__init__()
- self.init_arg_names = self._get_init_arg_names()
- self.generation_arg_names = self._get_generation_arg_names()
-
- def _get_init_arg_names(self) -> list[str]:
- """Get names of arguments for DiffGenerator initialization"""
- return ["num_gpus", "tp_size", "sp_size", "model_path"]
-
- def _get_generation_arg_names(self) -> list[str]:
- """Get names of arguments for generate_video method"""
- return [field.name for field in dataclasses.fields(SamplingParams)]
def cmd(
self, args: argparse.Namespace, unknown_args: list[str] | None = None
diff --git a/python/sglang/multimodal_gen/runtime/entrypoints/openai/protocol.py b/python/sglang/multimodal_gen/runtime/entrypoints/openai/protocol.py
index 13e8990084..73bec055fd 100644
--- a/python/sglang/multimodal_gen/runtime/entrypoints/openai/protocol.py
+++ b/python/sglang/multimodal_gen/runtime/entrypoints/openai/protocol.py
@@ -173,10 +173,6 @@ class VideoListResponse(BaseModel):
object: str = "list"
-class VideoRemixRequest(BaseModel):
- prompt: str
-
-
class RealtimeVideoGenerationsRequest(VideoGenerationsRequest):
enhance_prompt: Literal[False] = False
type: Literal["init"]
diff --git a/python/sglang/multimodal_gen/runtime/entrypoints/openai/realtime/adapters/sana_wm_realtime_adapter.py b/python/sglang/multimodal_gen/runtime/entrypoints/openai/realtime/adapters/sana_wm_realtime_adapter.py
index dd8e409484..f29a2cbcd5 100644
--- a/python/sglang/multimodal_gen/runtime/entrypoints/openai/realtime/adapters/sana_wm_realtime_adapter.py
+++ b/python/sglang/multimodal_gen/runtime/entrypoints/openai/realtime/adapters/sana_wm_realtime_adapter.py
@@ -38,7 +38,6 @@ if TYPE_CHECKING:
SANA_WM_DEFAULT_SIZE = "1280x704"
-SANA_WM_DEFAULT_NUM_FRAMES = 1081
SANA_WM_DEFAULT_FPS = 16
SANA_WM_DEFAULT_STEPS = 4
SANA_WM_DEFAULT_GUIDANCE = 1.0
diff --git a/python/sglang/multimodal_gen/runtime/layers/activation.py b/python/sglang/multimodal_gen/runtime/layers/activation.py
index e6a5357719..648057073c 100644
--- a/python/sglang/multimodal_gen/runtime/layers/activation.py
+++ b/python/sglang/multimodal_gen/runtime/layers/activation.py
@@ -41,9 +41,6 @@ class SiluAndMul(CustomOp):
return: (num_tokens, d) or (batch_size, seq_len, d)
"""
- def __init__(self) -> None:
- super().__init__()
-
def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
d = x.shape[-1] // 2
output_shape = x.shape[:-1] + (d,)
@@ -111,9 +108,6 @@ class GeluAndMul(CustomOp):
@CustomOp.register("gelu_new")
class NewGELU(CustomOp):
- def __init__(self):
- super().__init__()
-
def forward_cuda(self, *args, **kwargs) -> Any:
return self.forward_native(*args, **kwargs)
@@ -129,9 +123,6 @@ class NewGELU(CustomOp):
@CustomOp.register("quick_gelu")
class QuickGELU(CustomOp):
# https://github.com/huggingface/transformers/blob/main/src/transformers/activations.py#L90
- def __init__(self):
- super().__init__()
-
def forward_cuda(self, *args, **kwargs) -> Any:
return self.forward_native(*args, **kwargs)
diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/ascend_fa.py b/python/sglang/multimodal_gen/runtime/layers/attention/backends/ascend_fa.py
index a9f88cfc48..01b819bd5d 100644
--- a/python/sglang/multimodal_gen/runtime/layers/attention/backends/ascend_fa.py
+++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/ascend_fa.py
@@ -196,12 +196,6 @@ class AscendFAMetadata:
class AscendFAMetadataBuilder(AttentionMetadataBuilder):
- def __init__(self) -> None:
- pass
-
- def prepare(self) -> None:
- pass
-
def build(
self,
**kwargs: dict[str, Any],
diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/attention_backend.py b/python/sglang/multimodal_gen/runtime/layers/attention/backends/attention_backend.py
index a0326d2dc5..b4626cbe26 100644
--- a/python/sglang/multimodal_gen/runtime/layers/attention/backends/attention_backend.py
+++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/attention_backend.py
@@ -95,16 +95,6 @@ T = TypeVar("T", bound=AttentionMetadata)
class AttentionMetadataBuilder(ABC, Generic[T]):
"""Abstract class for attention metadata builders."""
- @abstractmethod
- def __init__(self) -> None:
- """Create the builder, remember some configuration and parameters."""
- raise NotImplementedError
-
- @abstractmethod
- def prepare(self) -> None:
- """Prepare for one batch."""
- raise NotImplementedError
-
@abstractmethod
def build(
self,
diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/block_sparse_attn.py b/python/sglang/multimodal_gen/runtime/layers/attention/backends/block_sparse_attn.py
index f51ffa9acc..419845c06c 100644
--- a/python/sglang/multimodal_gen/runtime/layers/attention/backends/block_sparse_attn.py
+++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/block_sparse_attn.py
@@ -53,12 +53,6 @@ class BlockSparseAttentionMetadata(AttentionMetadata):
class BlockSparseAttentionMetadataBuilder(AttentionMetadataBuilder):
- def __init__(self) -> None:
- pass
-
- def prepare(self) -> None:
- pass
-
def build(
self,
current_timestep: int,
diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/cube_sparse_attn/backend.py b/python/sglang/multimodal_gen/runtime/layers/attention/backends/cube_sparse_attn/backend.py
index 50c4d94ace..ef0b54786b 100644
--- a/python/sglang/multimodal_gen/runtime/layers/attention/backends/cube_sparse_attn/backend.py
+++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/cube_sparse_attn/backend.py
@@ -62,12 +62,6 @@ class CubeSparseAttentionMetadata(AttentionMetadata):
class CubeSparseAttentionMetadataBuilder(AttentionMetadataBuilder):
- def __init__(self):
- pass
-
- def prepare(self):
- pass
-
def build( # type: ignore[override]
self,
*,
diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/flash_attn.py b/python/sglang/multimodal_gen/runtime/layers/attention/backends/flash_attn.py
index 630677789a..0058204e46 100644
--- a/python/sglang/multimodal_gen/runtime/layers/attention/backends/flash_attn.py
+++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/flash_attn.py
@@ -322,12 +322,6 @@ class FlashAttentionMetadata:
class FlashAttentionMetadataBuilder(AttentionMetadataBuilder):
- def __init__(self) -> None:
- pass
-
- def prepare(self) -> None:
- pass
-
def build( # type: ignore
self,
raw_latent_shape=list,
@@ -386,7 +380,6 @@ class FlashAttentionImpl(AttentionImpl):
self.packed_trailing_padding = extra_impl_args.get(
"packed_trailing_padding", False
)
- self.attention_metadata = FlashAttentionMetadata()
def _request_skip_softmax_threshold(self) -> tuple[bool, float | None]:
params = get_request_skip_softmax_params()
diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/hybrid_window_attn_h3.py b/python/sglang/multimodal_gen/runtime/layers/attention/backends/hybrid_window_attn_h3.py
index 763e2eb067..f40d4d575c 100644
--- a/python/sglang/multimodal_gen/runtime/layers/attention/backends/hybrid_window_attn_h3.py
+++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/hybrid_window_attn_h3.py
@@ -268,12 +268,6 @@ class HybridWindowAttentionH3Metadata(AttentionMetadata):
class HybridWindowAttentionH3MetadataBuilder(AttentionMetadataBuilder):
- def __init__(self) -> None:
- pass
-
- def prepare(self) -> None:
- pass
-
def build( # type: ignore[override]
self,
*,
diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/rain_fusion_attn.py b/python/sglang/multimodal_gen/runtime/layers/attention/backends/rain_fusion_attn.py
index 48245874ea..583b07db8c 100644
--- a/python/sglang/multimodal_gen/runtime/layers/attention/backends/rain_fusion_attn.py
+++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/rain_fusion_attn.py
@@ -54,12 +54,6 @@ class RainFusionAttentionMetadata(AttentionMetadata):
class RainFusionAttentionMetadataBuilder(AttentionMetadataBuilder):
- def __init__(self) -> None:
- pass
-
- def prepare(self) -> None:
- pass
-
def build(
self,
current_timestep: int,
diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/sliding_tile_attn.py b/python/sglang/multimodal_gen/runtime/layers/attention/backends/sliding_tile_attn.py
index 17d7131a49..4c49773b39 100644
--- a/python/sglang/multimodal_gen/runtime/layers/attention/backends/sliding_tile_attn.py
+++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/sliding_tile_attn.py
@@ -80,12 +80,6 @@ class SlidingTileAttentionMetadata(AttentionMetadata):
class SlidingTileAttentionMetadataBuilder(AttentionMetadataBuilder):
- def __init__(self):
- pass
-
- def prepare(self):
- pass
-
def build( # type: ignore
self,
STA_param: list[list[Any]],
diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/sparse_linear_attn.py b/python/sglang/multimodal_gen/runtime/layers/attention/backends/sparse_linear_attn.py
index b3dcd6deec..827f7509b8 100644
--- a/python/sglang/multimodal_gen/runtime/layers/attention/backends/sparse_linear_attn.py
+++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/sparse_linear_attn.py
@@ -86,12 +86,6 @@ class SparseLinearAttentionMetadata(AttentionMetadata):
class SparseLinearAttentionMetadataBuilder(AttentionMetadataBuilder):
"""Builder for SparseLinearAttentionMetadata."""
- def __init__(self) -> None:
- pass
-
- def prepare(self) -> None:
- pass
-
def build(
self,
current_timestep: int,
@@ -104,6 +98,24 @@ class SparseLinearAttentionMetadataBuilder(AttentionMetadataBuilder):
)
+def _initialize_linear_attention(model, head_size: int, feature_map: str) -> None:
+ model.proj_l = nn.Linear(head_size, head_size, dtype=torch.float32)
+ model.feature_map_q: Callable[[torch.Tensor], torch.Tensor]
+ model.feature_map_k: Callable[[torch.Tensor], torch.Tensor]
+ if feature_map == "elu":
+ model.feature_map_q = lambda x: F.elu(x) + 1
+ model.feature_map_k = lambda x: F.elu(x) + 1
+ elif feature_map == "relu":
+ model.feature_map_q = F.relu
+ model.feature_map_k = F.relu
+ elif feature_map == "softmax":
+ model.feature_map_q = lambda x: F.softmax(x, dim=-1)
+ model.feature_map_k = lambda x: F.softmax(x, dim=-1)
+ else:
+ raise ValueError(f"Unknown feature map: {feature_map}")
+ model._init_weights()
+
+
class SparseLinearAttentionImpl(AttentionImpl, nn.Module):
"""Implementation of sparse linear attention for the backend."""
@@ -131,26 +143,7 @@ class SparseLinearAttentionImpl(AttentionImpl, nn.Module):
self.BLKK = BLKK
self.dtype = torch.bfloat16 if use_bf16 else torch.float16
- # Learnable linear projection for combining sparse + linear attention
- self.proj_l = nn.Linear(head_size, head_size, dtype=torch.float32)
-
- # Feature map for linear attention
- # Type annotation for callables
- self.feature_map_q: Callable[[torch.Tensor], torch.Tensor]
- self.feature_map_k: Callable[[torch.Tensor], torch.Tensor]
- if feature_map == "elu":
- self.feature_map_q = lambda x: F.elu(x) + 1
- self.feature_map_k = lambda x: F.elu(x) + 1
- elif feature_map == "relu":
- self.feature_map_q = F.relu
- self.feature_map_k = F.relu
- elif feature_map == "softmax":
- self.feature_map_q = lambda x: F.softmax(x, dim=-1)
- self.feature_map_k = lambda x: F.softmax(x, dim=-1)
- else:
- raise ValueError(f"Unknown feature map: {feature_map}")
-
- self._init_weights()
+ _initialize_linear_attention(self, head_size, feature_map)
def _init_weights(self) -> None:
"""Initialize projection weights to zero for residual-like behavior."""
@@ -322,12 +315,6 @@ class SageSparseLinearAttentionMetadata(AttentionMetadata):
class SageSparseLinearAttentionMetadataBuilder(AttentionMetadataBuilder):
"""Builder for SageSparseLinearAttentionMetadata."""
- def __init__(self) -> None:
- pass
-
- def prepare(self) -> None:
- pass
-
def build(
self,
current_timestep: int,
@@ -369,42 +356,12 @@ class SageSparseLinearAttentionImpl(AttentionImpl, nn.Module):
self.topk_ratio = topk_ratio
self.dtype = torch.bfloat16 if use_bf16 else torch.float16
- # Learnable linear projection for combining sparse + linear attention
- self.proj_l = nn.Linear(head_size, head_size, dtype=torch.float32)
+ _initialize_linear_attention(self, head_size, feature_map)
- # Feature map for linear attention
- # Type annotation for callables
- self.feature_map_q: Callable[[torch.Tensor], torch.Tensor]
- self.feature_map_k: Callable[[torch.Tensor], torch.Tensor]
- if feature_map == "elu":
- self.feature_map_q = lambda x: F.elu(x) + 1
- self.feature_map_k = lambda x: F.elu(x) + 1
- elif feature_map == "relu":
- self.feature_map_q = F.relu
- self.feature_map_k = F.relu
- elif feature_map == "softmax":
- self.feature_map_q = lambda x: F.softmax(x, dim=-1)
- self.feature_map_k = lambda x: F.softmax(x, dim=-1)
- else:
- raise ValueError(f"Unknown feature map: {feature_map}")
-
- self._init_weights()
-
- def _init_weights(self) -> None:
- """Initialize projection weights to zero for residual-like behavior."""
- with torch.no_grad():
- nn.init.zeros_(self.proj_l.weight)
- nn.init.zeros_(self.proj_l.bias) # type: ignore[arg-type]
-
- def _calc_linear_attention_with_torch(
- self,
- q: torch.Tensor,
- k: torch.Tensor,
- v: torch.Tensor,
- ):
- kv = torch.matmul(k.transpose(-1, -2), v)
- k_sum = torch.sum(k, dim=-2, keepdim=True)
- return torch.matmul(q, kv) / (1e-5 + torch.matmul(q, k_sum.transpose(-1, -2)))
+ _init_weights = SparseLinearAttentionImpl._init_weights
+ _calc_linear_attention_with_torch = (
+ SparseLinearAttentionImpl._calc_linear_attention_with_torch
+ )
def forward(
self,
diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/sparse_video_gen_2_attn.py b/python/sglang/multimodal_gen/runtime/layers/attention/backends/sparse_video_gen_2_attn.py
index f5d03072c1..77493349e9 100644
--- a/python/sglang/multimodal_gen/runtime/layers/attention/backends/sparse_video_gen_2_attn.py
+++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/sparse_video_gen_2_attn.py
@@ -106,21 +106,7 @@ class SparseVideoGen2AttentionMetadata(AttentionMetadata):
max_seqlen_k: int | None = None
-def _require_kwarg(kwargs: dict[str, Any], name: str) -> Any:
- if name not in kwargs:
- raise ValueError(
- f"Missing required argument for SparseVideoGen2Attention: {name}"
- )
- return kwargs[name]
-
-
class SparseVideoGen2AttentionMetadataBuilder(AttentionMetadataBuilder):
- def __init__(self) -> None:
- pass
-
- def prepare(self) -> None:
- pass
-
def build( # type: ignore[override]
self,
current_timestep: int,
diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/subblock_sparse_attn.py b/python/sglang/multimodal_gen/runtime/layers/attention/backends/subblock_sparse_attn.py
index e441fdd1c1..58b791f456 100644
--- a/python/sglang/multimodal_gen/runtime/layers/attention/backends/subblock_sparse_attn.py
+++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/subblock_sparse_attn.py
@@ -397,14 +397,6 @@ class SubBlockSparseAttentionMetadata(AttentionMetadata):
class SubBlockSparseAttentionMetadataBuilder(AttentionMetadataBuilder):
- # The base class declares __init__ abstract, so a builder that does not
- # override it cannot be instantiated at all.
- def __init__(self) -> None:
- pass
-
- def prepare(self) -> None:
- pass
-
def build( # type: ignore[override]
self, current_timestep: int, **kwargs: dict[str, Any]
) -> SubBlockSparseAttentionMetadata:
diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/video_sparse_attn.py b/python/sglang/multimodal_gen/runtime/layers/attention/backends/video_sparse_attn.py
index 6607321a82..6ae93233e2 100644
--- a/python/sglang/multimodal_gen/runtime/layers/attention/backends/video_sparse_attn.py
+++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/video_sparse_attn.py
@@ -174,12 +174,6 @@ def _compute_cur_topk(attn_metadata: VideoSparseAttentionMetadata) -> int:
class VideoSparseAttentionMetadataBuilder(AttentionMetadataBuilder):
- def __init__(self):
- pass
-
- def prepare(self):
- pass
-
def build( # type: ignore
self,
current_timestep: int,
@@ -189,7 +183,6 @@ class VideoSparseAttentionMetadataBuilder(AttentionMetadataBuilder):
device: torch.device,
**kwargs: dict[str, Any],
) -> VideoSparseAttentionMetadata:
- patch_size = patch_size
dit_seq_shape = (
raw_latent_shape[0] // patch_size[0],
raw_latent_shape[1] // patch_size[1],
diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/video_sparse_attn_h3.py b/python/sglang/multimodal_gen/runtime/layers/attention/backends/video_sparse_attn_h3.py
index aefb70f046..332c1d04a8 100644
--- a/python/sglang/multimodal_gen/runtime/layers/attention/backends/video_sparse_attn_h3.py
+++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/video_sparse_attn_h3.py
@@ -175,9 +175,6 @@ class VideoSparseAttentionH3MetadataBuilder(AttentionMetadataBuilder):
def __init__(self) -> None:
self._workspace_cache: dict = {}
- def prepare(self) -> None:
- pass
-
def build( # type: ignore[override]
self,
current_timestep: int,
diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/vmoba.py b/python/sglang/multimodal_gen/runtime/layers/attention/backends/vmoba.py
index 614c383076..193f228e58 100644
--- a/python/sglang/multimodal_gen/runtime/layers/attention/backends/vmoba.py
+++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/vmoba.py
@@ -88,12 +88,6 @@ def pad_input(hidden_states, indices, batch, seqlen):
class VideoMobaAttentionMetadataBuilder(AttentionMetadataBuilder):
- def __init__(self):
- pass
-
- def prepare(self):
- pass
-
def build( # type: ignore
self,
current_timestep: int,
diff --git a/python/sglang/multimodal_gen/runtime/layers/custom_op.py b/python/sglang/multimodal_gen/runtime/layers/custom_op.py
index 0fa1c3fb1c..88914ec549 100644
--- a/python/sglang/multimodal_gen/runtime/layers/custom_op.py
+++ b/python/sglang/multimodal_gen/runtime/layers/custom_op.py
@@ -129,14 +129,6 @@ class CustomOp(nn.Module):
# since we are not using Inductor, we always return True
return True
- @staticmethod
- def default_on() -> bool:
- """
- On by default if level < CompilationLevel.PIECEWISE
- Specifying 'all' or 'none' in custom_op takes precedence.
- """
- raise NotImplementedError
-
# Dictionary of all custom ops (classes, indexed by registered name).
# To check if an op with a name is enabled, call .enabled() on the class.
# Examples:
diff --git a/python/sglang/multimodal_gen/runtime/layers/linear.py b/python/sglang/multimodal_gen/runtime/layers/linear.py
index ec8ae6b68a..204d412644 100644
--- a/python/sglang/multimodal_gen/runtime/layers/linear.py
+++ b/python/sglang/multimodal_gen/runtime/layers/linear.py
@@ -88,6 +88,27 @@ def adjust_scalar_to_fused_array(
return param[shard_id], loaded_weight
+def _load_tensor_parallel_weight(
+ param: Parameter,
+ loaded_weight: torch.Tensor,
+ tp_rank: int,
+ shard_dim: int | None,
+) -> None:
+ is_sharded_weight = getattr(param, "is_sharded_weight", False)
+ param_data = param.data
+ # Already-sharded checkpoints (e.g. bitsandbytes) must not be sliced twice.
+ if shard_dim is not None and not is_sharded_weight:
+ shard_size = param_data.shape[shard_dim]
+ loaded_weight = loaded_weight.narrow(
+ shard_dim, tp_rank * shard_size, shard_size
+ )
+ # AutoFP8 scales can be stored as scalars rather than length-one tensors.
+ if len(loaded_weight.shape) == 0:
+ loaded_weight = loaded_weight.reshape(1)
+ assert param_data.shape == loaded_weight.shape
+ param_data.copy_(loaded_weight)
+
+
class LinearMethodBase(QuantizeMethodBase):
"""Base class for different (maybe quantized) linear methods."""
@@ -477,25 +498,9 @@ class ColumnParallelLinear(LinearBase):
self.register_parameter("bias", None)
def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor) -> None:
- tp_rank = self.tp_rank
- output_dim = getattr(param, "output_dim", None)
-
- is_sharded_weight = getattr(param, "is_sharded_weight", False)
- is_sharded_weight = is_sharded_weight
-
- param_data = param.data
- if output_dim is not None and not is_sharded_weight:
- shard_size = param_data.shape[output_dim]
- start_idx = tp_rank * shard_size
- loaded_weight = loaded_weight.narrow(output_dim, start_idx, shard_size)
-
- # Special case for loading scales off disk, which often do not
- # have a shape (such as in the case of AutoFP8).
- if len(loaded_weight.shape) == 0:
- loaded_weight = loaded_weight.reshape(1)
-
- assert param_data.shape == loaded_weight.shape
- param_data.copy_(loaded_weight)
+ _load_tensor_parallel_weight(
+ param, loaded_weight, self.tp_rank, getattr(param, "output_dim", None)
+ )
def weight_loader_v2(self, param: Parameter, loaded_weight: torch.Tensor) -> None:
# Special case for loading scales off disk, which often do not
@@ -629,7 +634,6 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
is_sharded_weight = getattr(param, "is_sharded_weight", False)
# bitsandbytes loads the weights of the specific portion
# no need to narrow
- is_sharded_weight = is_sharded_weight
param_data = param_data.narrow(output_dim, shard_offset, shard_size)
start_idx = tp_rank * shard_size
@@ -1027,7 +1031,6 @@ class QKVParallelLinear(ColumnParallelLinear):
is_sharded_weight = getattr(param, "is_sharded_weight", False)
# bitsandbytes loads the weights of the specific portion
# no need to narrow
- is_sharded_weight = is_sharded_weight
shard_idx = 0
param_data = param_data.narrow(output_dim, shard_offset, shard_size)
@@ -1151,26 +1154,9 @@ class RowParallelLinear(LinearBase):
self.register_parameter("bias", None)
def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor):
- tp_rank = self.tp_rank
- input_dim = getattr(param, "input_dim", None)
- is_sharded_weight = getattr(param, "is_sharded_weight", False)
- # bitsandbytes loads the weights of the specific portion
- # no need to narrow
- is_sharded_weight = is_sharded_weight
-
- param_data = param.data
- if input_dim is not None and not is_sharded_weight:
- shard_size = param_data.shape[input_dim]
- start_idx = tp_rank * shard_size
- loaded_weight = loaded_weight.narrow(input_dim, start_idx, shard_size)
-
- # Special case for loading scales off disk, which often do not
- # have a shape (such as in the case of AutoFP8).
- if len(loaded_weight.shape) == 0:
- loaded_weight = loaded_weight.reshape(1)
-
- assert param_data.shape == loaded_weight.shape
- param_data.copy_(loaded_weight)
+ _load_tensor_parallel_weight(
+ param, loaded_weight, self.tp_rank, getattr(param, "input_dim", None)
+ )
def weight_loader_v2(self, param: BasevLLMParameter, loaded_weight: torch.Tensor):
# Special case for loading scales off disk, which often do not
diff --git a/python/sglang/multimodal_gen/runtime/layers/lora/linear.py b/python/sglang/multimodal_gen/runtime/layers/lora/linear.py
index b8b3d4e749..1dca740132 100644
--- a/python/sglang/multimodal_gen/runtime/layers/lora/linear.py
+++ b/python/sglang/multimodal_gen/runtime/layers/lora/linear.py
@@ -32,9 +32,6 @@ from sglang.multimodal_gen.runtime.layers.linear import (
from sglang.multimodal_gen.runtime.layers.quantization.weight_only_fp8 import (
WeightOnlyFP8Linear,
)
-from sglang.multimodal_gen.runtime.layers.vocab_parallel_embedding import (
- VocabParallelEmbedding,
-)
from sglang.multimodal_gen.runtime.managers.forward_context import (
get_forward_context_or_none,
)
@@ -651,37 +648,7 @@ class BaseLayerWithLoRA(nn.Module):
self.strength = 1.0
-class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
- """
- Vocab parallel embedding layer with support for LoRA (Low-Rank Adaptation).
-
- Note: The current version does not yet implement the LoRA functionality.
- This class behaves exactly the same as the base VocabParallelEmbedding.
- Future versions will integrate LoRA functionality to support efficient parameter fine-tuning.
- """
-
- def __init__(
- self,
- base_layer: VocabParallelEmbedding,
- ) -> None:
- super().__init__(base_layer)
-
- def forward(self, input_: torch.Tensor) -> torch.Tensor:
- raise NotImplementedError(
- "We don't support VocabParallelEmbeddingWithLoRA yet."
- )
-
-
class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
- def __init__(
- self,
- base_layer: ColumnParallelLinear,
- lora_rank: int | None = None,
- lora_alpha: int | None = None,
- snapshot_base: bool = True,
- ) -> None:
- super().__init__(base_layer, lora_rank, lora_alpha, snapshot_base)
-
def forward(self, input_: torch.Tensor) -> torch.Tensor:
if self.disable_lora or (self.merged and not self.has_lora_output_offset):
return self.base_layer(input_)
@@ -738,15 +705,6 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
- def __init__(
- self,
- base_layer: MergedColumnParallelLinear,
- lora_rank: int | None = None,
- lora_alpha: int | None = None,
- snapshot_base: bool = True,
- ) -> None:
- super().__init__(base_layer, lora_rank, lora_alpha, snapshot_base)
-
def slice_lora_b_weights(self, B: torch.Tensor) -> torch.Tensor:
tp_rank = get_tp_rank()
if B.dim() == 3:
@@ -772,15 +730,6 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
class QKVParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
- def __init__(
- self,
- base_layer: QKVParallelLinear,
- lora_rank: int | None = None,
- lora_alpha: int | None = None,
- snapshot_base: bool = True,
- ) -> None:
- super().__init__(base_layer, lora_rank, lora_alpha, snapshot_base)
-
def slice_lora_b_weights(
self, B: list[torch.Tensor]
) -> tuple[torch.Tensor, torch.Tensor]:
@@ -802,15 +751,6 @@ class QKVParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
- def __init__(
- self,
- base_layer: RowParallelLinear,
- lora_rank: int | None = None,
- lora_alpha: int | None = None,
- snapshot_base: bool = True,
- ) -> None:
- super().__init__(base_layer, lora_rank, lora_alpha, snapshot_base)
-
def forward(self, input_: torch.Tensor):
if self.disable_lora or (self.merged and not self.has_lora_output_offset):
return self.base_layer(input_)
@@ -890,15 +830,6 @@ class LinearWithLoRA(BaseLayerWithLoRA):
not a tuple of (output, bias).
"""
- def __init__(
- self,
- base_layer: nn.Linear,
- lora_rank: int | None = None,
- lora_alpha: int | None = None,
- snapshot_base: bool = True,
- ) -> None:
- super().__init__(base_layer, lora_rank, lora_alpha, snapshot_base)
-
@torch.compile()
def forward(self, x: torch.Tensor) -> torch.Tensor:
lora_A = self.lora_A
@@ -989,7 +920,6 @@ def wrap_with_lora_layer(
"""
supported_layer_types: dict[type[nn.Module], type[BaseLayerWithLoRA]] = {
# the order matters
- # VocabParallelEmbedding: VocabParallelEmbeddingWithLoRA,
# Weight-only FP8 LoRA is currently dynamic-only and intended for
# single-GPU deployments.
WeightOnlyFP8Linear: WeightOnlyFP8LinearWithLoRA,
diff --git a/python/sglang/multimodal_gen/runtime/layers/parallel_conv.py b/python/sglang/multimodal_gen/runtime/layers/parallel_conv.py
index f1b45d781a..3b2081566c 100644
--- a/python/sglang/multimodal_gen/runtime/layers/parallel_conv.py
+++ b/python/sglang/multimodal_gen/runtime/layers/parallel_conv.py
@@ -181,7 +181,6 @@ def split_height_for_parallel_decode(
def _maybe_contiguous_for_sp_gather(x: torch.Tensor) -> torch.Tensor:
if (
x.dim() == 5
- and hasattr(torch, "channels_last_3d")
and x.is_contiguous(memory_format=torch.channels_last_3d)
and not x.is_contiguous()
):
@@ -302,7 +301,7 @@ def gather_variable_height(
def _halo_memory_format(reference: torch.Tensor) -> torch.memory_format:
if reference.dim() > 1 and reference.stride(1) == 1:
- if reference.dim() == 5 and hasattr(torch, "channels_last_3d"):
+ if reference.dim() == 5:
return torch.channels_last_3d
if reference.dim() == 4:
return torch.channels_last
@@ -475,7 +474,6 @@ def _conv_preserves_local_height(
def _conv3d_weight_is_channels_last_3d(weight: torch.Tensor) -> bool:
return (
weight.dim() == 5
- and hasattr(torch, "channels_last_3d")
and (current_platform.is_cuda() or current_platform.is_rocm())
and weight.is_contiguous(memory_format=torch.channels_last_3d)
)
diff --git a/python/sglang/multimodal_gen/runtime/layers/quantization/convrot_int8_comfy_kitchen.py b/python/sglang/multimodal_gen/runtime/layers/quantization/convrot_int8_comfy_kitchen.py
index b33a9c1eb4..435893910c 100644
--- a/python/sglang/multimodal_gen/runtime/layers/quantization/convrot_int8_comfy_kitchen.py
+++ b/python/sglang/multimodal_gen/runtime/layers/quantization/convrot_int8_comfy_kitchen.py
@@ -21,7 +21,10 @@ import os
import torch
from torch.nn.parameter import Parameter
-from sglang.multimodal_gen.runtime.layers.linear import LinearMethodBase
+from sglang.multimodal_gen.runtime.layers.linear import (
+ LinearMethodBase,
+ UnquantizedLinearMethod,
+)
from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import (
ConvRotInt8Config,
)
@@ -118,20 +121,17 @@ class ConvRotInt8ComfyKitchenLinearMethod(LinearMethodBase):
"--quantization-ignored-layers"
)
- # The online path initially matches UnquantizedLinearMethod so the
- # source weights load in BF16 before quantization. Serialized weights
- # allocate their final INT8 storage immediately.
- weight = Parameter(
- torch.empty(
- sum(output_partition_sizes),
- input_size_per_partition,
- dtype=(torch.int8 if self.is_checkpoint_serialized else params_dtype),
- ),
- requires_grad=False,
+ # Online checkpoints load their original dtype; serialized ones use INT8.
+ UnquantizedLinearMethod.create_weights(
+ self,
+ layer,
+ input_size_per_partition,
+ output_partition_sizes,
+ input_size,
+ output_size,
+ torch.int8 if self.is_checkpoint_serialized else params_dtype,
+ **extra_weight_attrs,
)
- set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0})
- layer.register_parameter("weight", weight)
- set_weight_attrs(weight, extra_weight_attrs)
if self.is_checkpoint_serialized:
weight_scale = Parameter(
torch.empty(
diff --git a/python/sglang/multimodal_gen/runtime/layers/quantization/convrot_int8_jit.py b/python/sglang/multimodal_gen/runtime/layers/quantization/convrot_int8_jit.py
index 5e4469246f..5831c3884d 100644
--- a/python/sglang/multimodal_gen/runtime/layers/quantization/convrot_int8_jit.py
+++ b/python/sglang/multimodal_gen/runtime/layers/quantization/convrot_int8_jit.py
@@ -43,6 +43,7 @@ from sglang.multimodal_gen.runtime.layers.linear import (
LinearMethodBase,
ReplicatedLinear,
RowParallelLinear,
+ UnquantizedLinearMethod,
)
from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import (
ConvRotInt8Config,
@@ -202,20 +203,17 @@ class ConvRotInt8JitLinearMethod(LinearMethodBase):
f"convrot_int8 backend jit needs the output size per "
f"partition ({sum(output_partition_sizes)}) to be a multiple of 8"
)
- # The online path initially matches UnquantizedLinearMethod so the
- # source weights load in BF16 before quantization. Serialized weights
- # allocate their final INT8 storage immediately.
- weight = Parameter(
- torch.empty(
- sum(output_partition_sizes),
- input_size_per_partition,
- dtype=(torch.int8 if self.is_checkpoint_serialized else params_dtype),
- ),
- requires_grad=False,
+ # Online checkpoints load their original dtype; serialized ones use INT8.
+ UnquantizedLinearMethod.create_weights(
+ self,
+ layer,
+ input_size_per_partition,
+ output_partition_sizes,
+ input_size,
+ output_size,
+ torch.int8 if self.is_checkpoint_serialized else params_dtype,
+ **extra_weight_attrs,
)
- set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0})
- layer.register_parameter("weight", weight)
- set_weight_attrs(weight, extra_weight_attrs)
if self.is_checkpoint_serialized:
# Comfy stores the per-row scale as [N, 1]; the ops read N contiguous
# floats and accept that shape unchanged.
diff --git a/python/sglang/multimodal_gen/runtime/layers/quantization/modelopt_quant.py b/python/sglang/multimodal_gen/runtime/layers/quantization/modelopt_quant.py
index 1effce239a..2e784e31c6 100755
--- a/python/sglang/multimodal_gen/runtime/layers/quantization/modelopt_quant.py
+++ b/python/sglang/multimodal_gen/runtime/layers/quantization/modelopt_quant.py
@@ -21,6 +21,9 @@ from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config impor
from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import (
ConvRotInt8Config,
)
+from sglang.multimodal_gen.runtime.layers.quantization.modelopt_fp8 import (
+ ModelOptFp8LinearMethod as StaticModelOptFp8LinearMethod,
+)
from sglang.multimodal_gen.runtime.models.parameter import (
ModelWeightParameter,
PerTensorScaleParameter,
@@ -28,7 +31,6 @@ from sglang.multimodal_gen.runtime.models.parameter import (
from sglang.multimodal_gen.runtime.platforms import current_platform
from sglang.multimodal_gen.runtime.utils.weight_attrs import set_weight_attrs
from sglang.srt.layers.quantization.fp8_utils import (
- apply_fp8_linear,
cutlass_fp8_supported,
normalize_e4m3fn_to_e4m3fnuz,
)
@@ -578,20 +580,7 @@ class ModelOptFp8LinearMethod(LinearMethodBase):
copy_or_rebind_param(layer, "weight_scale", processed_weight_scale)
copy_or_rebind_param(layer, "input_scale", layer.input_scale.max())
- def apply(
- self,
- layer: torch.nn.Module,
- x: torch.Tensor,
- bias: Optional[torch.Tensor] = None,
- ) -> torch.Tensor:
- return apply_fp8_linear(
- input=x,
- weight=layer.weight,
- weight_scale=layer.weight_scale,
- input_scale=layer.input_scale,
- bias=bias,
- cutlass_fp8_supported=self.cutlass_fp8_supported,
- )
+ apply = StaticModelOptFp8LinearMethod.apply
class ModelOptFp4LinearMethod(LinearMethodBase):
@@ -820,33 +809,9 @@ class ModelOptFp4LinearMethod(LinearMethodBase):
)
x_fp4, x_scale_interleaved = fp4_quantize(x, layer.input_scale_inv)
- weights_padding_cols = getattr(layer, "weights_padding_cols", 0)
- x_fp4 = pad_nvfp4_activation_for_cutlass(x_fp4, weights_padding_cols)
-
- w = layer.weight
- w_scale_interleaved = layer.weight_scale_interleaved
-
- if x_scale_interleaved.dtype == torch.uint8:
- x_scale_interleaved = x_scale_interleaved.view(torch.float8_e4m3fn)
- if w_scale_interleaved.dtype == torch.uint8:
- w_scale_interleaved = w_scale_interleaved.view(torch.float8_e4m3fn)
- fp4_gemm, flashinfer_backend = _get_fp4_gemm_op()
- if fp4_gemm is None:
- raise RuntimeError("No FP4 GEMM kernel available. Install flashinfer.")
- out = fp4_gemm(
- x_fp4,
- w.T,
- x_scale_interleaved,
- w_scale_interleaved.T,
- layer.alpha,
- output_dtype,
- backend=flashinfer_backend,
+ out = apply_nvfp4_gemm_prequantized(
+ layer, x_fp4, x_scale_interleaved, output_dtype, bias
)
-
- out = slice_nvfp4_output(out, output_size)
-
- if bias is not None:
- out = out + bias
return out.view(*output_shape)
diff --git a/python/sglang/multimodal_gen/runtime/layers/quantization/mxfp4_npu.py b/python/sglang/multimodal_gen/runtime/layers/quantization/mxfp4_npu.py
index 7cb9bd3118..f8c8cb395d 100644
--- a/python/sglang/multimodal_gen/runtime/layers/quantization/mxfp4_npu.py
+++ b/python/sglang/multimodal_gen/runtime/layers/quantization/mxfp4_npu.py
@@ -44,9 +44,6 @@ logger = init_logger(__name__)
class NPUMXFP4Config(QuantizationConfig):
"""Config for online MXFP4 quantization on NPU (Diffusion)."""
- def __init__(self) -> None:
- super().__init__()
-
@classmethod
def get_name(cls) -> str:
return "mxfp4_npu"
diff --git a/python/sglang/multimodal_gen/runtime/layers/rotary_embedding/factory.py b/python/sglang/multimodal_gen/runtime/layers/rotary_embedding/factory.py
index f039782647..fcffe9e073 100644
--- a/python/sglang/multimodal_gen/runtime/layers/rotary_embedding/factory.py
+++ b/python/sglang/multimodal_gen/runtime/layers/rotary_embedding/factory.py
@@ -10,7 +10,6 @@ from .mrope import NDRotaryEmbedding, _to_tuple
_ROPE_DICT: dict[tuple, RotaryEmbedding] = {}
_ND_ROPE_CACHE: "OrderedDict[tuple, NDRotaryEmbedding]" = OrderedDict()
-_ROPE_3D_CACHE: "OrderedDict[tuple, tuple[torch.Tensor, torch.Tensor]]" = OrderedDict()
def get_rope(
diff --git a/python/sglang/multimodal_gen/runtime/layers/rotary_embedding/mrope.py b/python/sglang/multimodal_gen/runtime/layers/rotary_embedding/mrope.py
index 2981393bf0..c6b738b131 100644
--- a/python/sglang/multimodal_gen/runtime/layers/rotary_embedding/mrope.py
+++ b/python/sglang/multimodal_gen/runtime/layers/rotary_embedding/mrope.py
@@ -367,6 +367,26 @@ class NDRotaryEmbedding(torch.nn.Module):
positions = torch.tensor(pos_tuple, dtype=torch.long, device=device)
return self.forward_uncached(pos=positions)
+ def forward_3d_sequence_shard(
+ self,
+ local_len: int,
+ rank: int,
+ frame_stride_local: int,
+ width_local: int,
+ device: torch.device,
+ ) -> tuple[torch.Tensor, torch.Tensor]:
+ """Embed a contiguous shard of a flattened (time, height, width) grid."""
+ token_start = rank * local_len
+ token_indices = torch.arange(
+ token_start, token_start + local_len, device=device, dtype=torch.long
+ )
+ t_idx = token_indices // frame_stride_local
+ rem = token_indices % frame_stride_local
+ h_idx = rem // width_local
+ w_idx = rem % width_local
+ positions = torch.stack((t_idx, h_idx, w_idx), dim=1)
+ return self.forward_uncached(positions)
+
def forward_uncached(self, pos: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
"""
The core implementation that computes embeddings from a position tensor.
diff --git a/python/sglang/multimodal_gen/runtime/layers/usp.py b/python/sglang/multimodal_gen/runtime/layers/usp.py
index 487a8874ff..6be91d9404 100644
--- a/python/sglang/multimodal_gen/runtime/layers/usp.py
+++ b/python/sglang/multimodal_gen/runtime/layers/usp.py
@@ -6,7 +6,6 @@ from typing import TYPE_CHECKING
import torch
import torch.distributed as dist
-import torch.distributed._functional_collectives as ft_c
from torch.distributed.tensor.experimental._attention import _cp_options
from sglang.kernels.ops.diffusion import pack_qkv_destination_major, usp_merge_heads
@@ -28,16 +27,6 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
-def _maybe_wait(tensor: torch.Tensor) -> torch.Tensor:
- """
- When tracing the code, the result tensor is not an AsyncCollectiveTensor,
- so we cannot call ``wait()``.
- """
- if isinstance(tensor, ft_c.AsyncCollectiveTensor):
- return tensor.wait()
- return tensor
-
-
_A2A_STAGING_BUFFERS: dict[tuple[str, torch.dtype, int], torch.Tensor] = {}
diff --git a/python/sglang/multimodal_gen/runtime/layers/utils.py b/python/sglang/multimodal_gen/runtime/layers/utils.py
index 58e2416ca8..291c2005e1 100644
--- a/python/sglang/multimodal_gen/runtime/layers/utils.py
+++ b/python/sglang/multimodal_gen/runtime/layers/utils.py
@@ -79,13 +79,7 @@ def direct_register_custom_op(
# Operator doesn't exist, proceed with registration
pass
- if hasattr(torch.library, "infer_schema"):
- schema_str = torch.library.infer_schema(op_func, mutates_args=mutates_args)
- else:
- # for pytorch 2.4
- import torch._custom_op.impl
-
- schema_str = torch._custom_op.impl.infer_schema(op_func, mutates_args)
+ schema_str = torch.library.infer_schema(op_func, mutates_args=mutates_args)
try:
my_lib.define(op_name + schema_str)
@@ -106,9 +100,6 @@ def direct_register_custom_op(
else:
# Re-raise other RuntimeErrors
raise error
- except AttributeError as error:
- # Always re-raise AttributeError as it indicates missing dependencies
- raise error
class CustomOpWrapper:
diff --git a/python/sglang/multimodal_gen/runtime/layers/vocab_parallel_embedding.py b/python/sglang/multimodal_gen/runtime/layers/vocab_parallel_embedding.py
index 050f9e9aca..565c2f77f6 100644
--- a/python/sglang/multimodal_gen/runtime/layers/vocab_parallel_embedding.py
+++ b/python/sglang/multimodal_gen/runtime/layers/vocab_parallel_embedding.py
@@ -15,6 +15,7 @@ from sglang.multimodal_gen.runtime.distributed import (
get_tp_group,
tensor_model_parallel_all_reduce,
)
+from sglang.multimodal_gen.runtime.layers.linear import UnquantizedLinearMethod
from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config import (
QuantizationConfig,
QuantizeMethodBase,
@@ -23,7 +24,6 @@ from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config impor
from sglang.multimodal_gen.runtime.layers.utils import get_group_rank, get_group_size
from sglang.multimodal_gen.runtime.models.parameter import BasevLLMParameter
from sglang.multimodal_gen.runtime.platforms import current_platform
-from sglang.multimodal_gen.runtime.utils.weight_attrs import set_weight_attrs
DEFAULT_VOCAB_PADDING_SIZE = 64
@@ -31,29 +31,7 @@ DEFAULT_VOCAB_PADDING_SIZE = 64
class UnquantizedEmbeddingMethod(QuantizeMethodBase):
"""Unquantized method for embeddings."""
- def create_weights(
- self,
- layer: torch.nn.Module,
- input_size_per_partition: int,
- output_partition_sizes: list[int],
- input_size: int,
- output_size: int,
- params_dtype: torch.dtype,
- **extra_weight_attrs,
- ):
- """Create weights for embedding layer."""
-
- weight = Parameter(
- torch.empty(
- sum(output_partition_sizes),
- input_size_per_partition,
- dtype=params_dtype,
- ),
- requires_grad=False,
- )
- set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0})
- layer.register_parameter("weight", weight)
- set_weight_attrs(weight, extra_weight_attrs)
+ create_weights = UnquantizedLinearMethod.create_weights
def apply(
self, layer: torch.nn.Module, x: torch.Tensor, bias: torch.Tensor | None = None
@@ -344,71 +322,6 @@ class VocabParallelEmbedding(torch.nn.Module):
added_vocab_end_index,
)
- def get_sharded_to_full_mapping(self) -> list[int] | None:
- """Get a mapping that can be used to reindex the gathered
- logits for sampling.
-
- During sampling, we gather logits from all ranks. The relationship
- of index->token_id will follow the same format as outlined in the class
- docstring. However, after the gather, we want to reindex the final
- logits tensor to map index->token_id one-to-one (the index is always
- equal the token_id it corresponds to). The indices returned by this
- method allow us to do that.
- """
- if self.tp_size < 2:
- return None
-
- base_embeddings: list[int] = []
- added_embeddings: list[int] = []
- padding: list[int] = []
- for tp_rank in range(self.tp_size):
- shard_indices = self._get_indices(
- self.num_embeddings_padded,
- self.org_vocab_size_padded,
- self.num_embeddings,
- self.org_vocab_size,
- tp_rank,
- self.tp_size,
- )
- range_start = self.num_embeddings_per_partition * tp_rank
- range_end = self.num_embeddings_per_partition * (tp_rank + 1)
- base_embeddings.extend(
- range(range_start, range_start + shard_indices.num_org_elements)
- )
- padding.extend(
- range(
- range_start + shard_indices.num_org_elements,
- range_start + shard_indices.num_org_elements_padded,
- )
- )
- added_embeddings.extend(
- range(
- range_start + shard_indices.num_org_elements_padded,
- range_start
- + shard_indices.num_org_elements_padded
- + shard_indices.num_added_elements,
- )
- )
- padding.extend(
- range(
- range_start
- + shard_indices.num_org_elements_padded
- + shard_indices.num_added_elements,
- range_start
- + shard_indices.num_org_elements_padded
- + shard_indices.num_added_elements_padded,
- )
- )
- assert (
- range_start
- + shard_indices.num_org_elements_padded
- + shard_indices.num_added_elements_padded
- == range_end
- )
- ret = base_embeddings + added_embeddings + padding
- assert len(ret) == self.num_embeddings_padded
- return ret
-
def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor):
output_dim = getattr(param, "output_dim", None)
packed_dim = getattr(param, "packed_dim", None)
diff --git a/python/sglang/multimodal_gen/runtime/loader/component_loaders/text_encoder_loader.py b/python/sglang/multimodal_gen/runtime/loader/component_loaders/text_encoder_loader.py
index 0ef27fb542..fd6147e402 100644
--- a/python/sglang/multimodal_gen/runtime/loader/component_loaders/text_encoder_loader.py
+++ b/python/sglang/multimodal_gen/runtime/loader/component_loaders/text_encoder_loader.py
@@ -5,7 +5,7 @@ from collections.abc import Generator
import torch
import transformers
from torch import nn
-from transformers import PretrainedConfig
+from transformers import AutoConfig, PretrainedConfig
from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
from sglang.multimodal_gen.configs.models import EncoderConfig
@@ -83,7 +83,6 @@ from sglang.multimodal_gen.runtime.models.registry import ModelRegistry
from sglang.multimodal_gen.runtime.platforms import current_platform
from sglang.multimodal_gen.runtime.server_args import ServerArgs
from sglang.multimodal_gen.runtime.utils.hf_diffusers_utils import (
- get_config,
get_diffusers_component_config,
load_dict,
)
@@ -568,7 +567,7 @@ class TextEncoderLoader(OnlineQuantizationComponentLoader):
server_args: ServerArgs,
component_name: str,
) -> EncoderConfig:
- diffusers_pretrained_config = get_config(
+ diffusers_pretrained_config = AutoConfig.from_pretrained(
component_model_path, trust_remote_code=True
)
encoder_index = self._extract_encoder_index(
diff --git a/python/sglang/multimodal_gen/runtime/loader/component_loaders/vae_loader.py b/python/sglang/multimodal_gen/runtime/loader/component_loaders/vae_loader.py
index 892248b7dc..620e84a3c6 100644
--- a/python/sglang/multimodal_gen/runtime/loader/component_loaders/vae_loader.py
+++ b/python/sglang/multimodal_gen/runtime/loader/component_loaders/vae_loader.py
@@ -118,8 +118,6 @@ def _convert_conv3d_weights_to_channels_last_3d(module: nn.Module) -> int:
Convert Conv3d weights to channels_last_3d (NDHWC) memory format.
Returns the number of Conv3d modules converted.
"""
- if not hasattr(torch, "channels_last_3d"):
- return 0
num_converted = 0
for m in module.modules():
if isinstance(m, nn.Conv3d):
diff --git a/python/sglang/multimodal_gen/runtime/loader/weight_utils.py b/python/sglang/multimodal_gen/runtime/loader/weight_utils.py
index 831d6e0347..6ae0d4f0a7 100644
--- a/python/sglang/multimodal_gen/runtime/loader/weight_utils.py
+++ b/python/sglang/multimodal_gen/runtime/loader/weight_utils.py
@@ -5,7 +5,6 @@
"""Utilities for downloading, loading, initializing and verifying model weights."""
import hashlib
-import json
import os
import tempfile
from collections import defaultdict
@@ -139,38 +138,6 @@ def get_lock(model_name_or_path: str | Path, cache_dir: str | None = None):
return lock
-# For models like Mistral-7B-v0.3, there are both sharded
-# safetensors files and a consolidated safetensors file.
-# Passing both of these to the weight loader functionality breaks.
-# So, we use the index_file to
-# look up which safetensors files should be used.
-def filter_duplicate_safetensors_files(
- hf_weights_files: list[str],
- hf_folder: str,
- index_file: str,
- key_filter: Callable[[str], bool] | None = None,
-) -> list[str]:
- # model.safetensors.index.json is a mapping from keys in the
- # torch state_dict to safetensors file holding that weight.
- index_file_name = os.path.join(hf_folder, index_file)
- if not os.path.isfile(index_file_name):
- return hf_weights_files
-
- # Iterate through the weight_map (weight_name: safetensors files)
- # to identify weights that we should use.
- with open(index_file_name) as f:
- weight_map = json.load(f)["weight_map"]
- weight_files_in_index = set()
- for weight_name in weight_map:
- # remove only shards whose indexed tensors are all filtered
- if key_filter is not None and not key_filter(weight_name):
- continue
- weight_files_in_index.add(os.path.join(hf_folder, weight_map[weight_name]))
- # Filter out any fields that are not found in the index file.
- hf_weights_files = [f for f in hf_weights_files if f in weight_files_in_index]
- return hf_weights_files
-
-
def filter_files_not_needed_for_inference(hf_weights_files: list[str]) -> list[str]:
"""
Exclude files that are not needed for inference.
@@ -376,23 +343,18 @@ def pt_weights_iterator(
def default_weight_loader(param: torch.Tensor, loaded_weight: torch.Tensor) -> None:
"""Default weight loader."""
- try:
- if param.numel() == 1 and loaded_weight.numel() == 1:
- # Sometimes scalar values aren't considered tensors with shapes
- # so if both param and loaded_weight are a scalar,
- # "broadcast" instead of copy
- param.data.fill_(loaded_weight.item())
- else:
- assert param.size() == loaded_weight.size(), (
- f"Attempted to load weight ({loaded_weight.size()}) "
- f"into parameter ({param.size()})"
- )
+ if param.numel() == 1 and loaded_weight.numel() == 1:
+ # Sometimes scalar values aren't considered tensors with shapes
+ # so if both param and loaded_weight are a scalar,
+ # "broadcast" instead of copy
+ param.data.fill_(loaded_weight.item())
+ else:
+ assert param.size() == loaded_weight.size(), (
+ f"Attempted to load weight ({loaded_weight.size()}) "
+ f"into parameter ({param.size()})"
+ )
- param.data.copy_(loaded_weight)
- except Exception:
- # NOTE: This exception is added for the purpose of setting breakpoint to
- # debug weight loading issues.
- raise
+ param.data.copy_(loaded_weight)
def load_stacked_weight(
@@ -485,6 +447,36 @@ def maybe_remap_kv_scale_name(name: str, params_dict: dict) -> str | None:
return name
+def load_llm_encoder_weights(
+ weights: Iterable[tuple[str, torch.Tensor]],
+ params_dict: dict[str, torch.nn.Parameter],
+ stacked_params_mapping: Iterable[tuple[str, str, str | int]],
+ *,
+ strip_prefix: str = "",
+) -> set[str]:
+ """Load Llama/Qwen-style encoder weights, excluding rotary cache tensors."""
+ loaded_params: set[str] = set()
+ for name, loaded_weight in weights:
+ if strip_prefix and name.startswith(strip_prefix):
+ name = name[len(strip_prefix) :]
+ if (
+ "rotary_emb.inv_freq" in name
+ or "rotary_emb.cos_cached" in name
+ or "rotary_emb.sin_cached" in name
+ ):
+ continue
+ if "scale" in name:
+ name = maybe_remap_kv_scale_name(name, params_dict)
+ if name is None:
+ continue
+ name = load_stacked_weight(
+ name, loaded_weight, params_dict, stacked_params_mapping
+ )
+ if name is not None:
+ loaded_params.add(name)
+ return loaded_params
+
+
def compute_weights_checksum(
named_params: Iterable[tuple[str, torch.Tensor]],
) -> str:
diff --git a/python/sglang/multimodal_gen/runtime/managers/forward_context.py b/python/sglang/multimodal_gen/runtime/managers/forward_context.py
index f59e1524c3..4fbfd626ae 100644
--- a/python/sglang/multimodal_gen/runtime/managers/forward_context.py
+++ b/python/sglang/multimodal_gen/runtime/managers/forward_context.py
@@ -2,51 +2,20 @@
# SPDX-License-Identifier: Apache-2.0
# Adapted from vllm: https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/forward_context.py
-import time
-from collections import defaultdict
from contextlib import contextmanager
from dataclasses import dataclass
-from typing import TYPE_CHECKING, Optional, Type
-
-import torch
-
-from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
+from typing import TYPE_CHECKING, Optional
if TYPE_CHECKING:
from sglang.multimodal_gen.runtime.layers.attention import AttentionMetadata
from sglang.multimodal_gen.runtime.pipelines_core import Req
-logger = init_logger(__name__)
-
-# TODO(will): check if this is needed
-# track_batchsize: bool = envs.SGLANG_DIFFUSION_LOG_BATCHSIZE_INTERVAL >= 0
-track_batchsize: bool = False
-last_logging_time: float = 0
-forward_start_time: float = 0
-# batchsize_logging_interval: float = envs.SGLANG_DIFFUSION_LOG_BATCHSIZE_INTERVAL
-batchsize_logging_interval: float = 1000
-batchsize_forward_time: defaultdict = defaultdict(list)
-
@dataclass
class ForwardContext:
current_timestep: int
- # TODO(will): check this arg
- # copy from vllm_config.compilation_config.static_forward_context
- # attn_layers: Dict[str, Any]
- # TODO: extend to support per-layer dynamic forward context
attn_metadata: "AttentionMetadata" # set dynamically for each forward pass
forward_batch: Optional["Req"] = None
- attention_backend_cls: Optional[Type] = None
-
- def set_attn_backend_cls(self, attention_backend_cls: Type):
- if self.attention_backend_cls:
- if self.attention_backend_cls != attention_backend_cls:
- raise RuntimeError(
- f"Different types of attention backend in a same context detected, previous: {self.attention_backend_cls}, new: {attention_backend_cls}"
- )
- else:
- self.attention_backend_cls = attention_backend_cls
_forward_context: Optional["ForwardContext"] = None
@@ -65,7 +34,6 @@ def get_forward_context_or_none() -> "ForwardContext | None":
return _forward_context
-# TODO(will): finalize the interface
@contextmanager
def set_forward_context(
current_timestep, attn_metadata, forward_batch: Optional["Req"] = None
@@ -74,10 +42,6 @@ def set_forward_context(
can be attention metadata, etc.
Here we can inject common logic for every model forward pass.
"""
- global forward_start_time
- need_to_track_batchsize = track_batchsize and attn_metadata is not None
- if need_to_track_batchsize:
- forward_start_time = time.perf_counter()
global _forward_context
prev_context = _forward_context
_forward_context = ForwardContext(
@@ -89,36 +53,4 @@ def set_forward_context(
try:
yield
finally:
- global last_logging_time, batchsize_logging_interval
- if need_to_track_batchsize:
- if hasattr(attn_metadata, "num_prefill_tokens"):
- # for v0 attention backends
- batchsize = (
- attn_metadata.num_prefill_tokens + attn_metadata.num_decode_tokens
- )
- else:
- # for v1 attention backends
- batchsize = attn_metadata.num_input_tokens
- now = time.perf_counter()
- # time measurement is in milliseconds
- batchsize_forward_time[batchsize].append((now - forward_start_time) * 1000)
- if now - last_logging_time > batchsize_logging_interval:
- last_logging_time = now
- forward_stats = []
- for bs, times in batchsize_forward_time.items():
- if len(times) <= 1:
- # can be cudagraph / profiling run
- continue
- medium = torch.quantile(torch.tensor(times), q=0.5).item()
- medium = round(medium, 2)
- forward_stats.append((bs, len(times), medium))
- forward_stats.sort(key=lambda x: x[1], reverse=True)
- if forward_stats:
- logger.info(
- (
- "Batchsize forward time stats "
- "(batchsize, count, median_time(ms)): %s"
- ),
- forward_stats,
- )
_forward_context = prev_context
diff --git a/python/sglang/multimodal_gen/runtime/managers/memory_managers/auto_residency.py b/python/sglang/multimodal_gen/runtime/managers/memory_managers/auto_residency.py
index 935904eee5..22f3700bfa 100644
--- a/python/sglang/multimodal_gen/runtime/managers/memory_managers/auto_residency.py
+++ b/python/sglang/multimodal_gen/runtime/managers/memory_managers/auto_residency.py
@@ -1,25 +1,9 @@
# SPDX-License-Identifier: Apache-2.0
-"""Warmup-calibrated automatic component residency placement.
+"""Warmup memory, timing and layer-use estimates for the default workload.
-Under ``--performance-mode auto`` with server warmup, each rank measures the
-peak GPU memory of bounded synthetic warmup requests and a low-step probe at
-the complete default serving shape, then selects a complete serving placement
-for every eligible component under the measured memory constraints.
-
-When no full-shape measurement is available, the fallback estimate splits the
-measured peak into persistent weights and workload-scaled activations. Scaling
-the whole peak would multiply resident weights by the video frame/area cap
-ratio (~16x for Wan-class defaults) and residency adjustment would never trigger.
-
-The planner targets the model default workload only (default resolution,
-default frames, batch=1). Larger shapes, batches, or multi-image inputs need
-explicit ``--component-residency``.
-
-Loading and serving are deliberately separate placement states. The existing
-auto policy provides the initial state; when that state can complete loading
-and calibration, this module optimizes the long-lived serving state and
-validates the transition with a post-placement warmup. It does not force a
-single placement to serve two different lifecycle objectives.
+Peak-memory extrapolation separates persistent weights from workload-scaled
+activations. Phase and layer-use estimates account for stage-specific iteration
+counts rather than scaling every stage by the request's denoising steps.
"""
from __future__ import annotations
@@ -29,61 +13,15 @@ from typing import Iterable, Mapping
import msgspec
-from sglang.multimodal_gen.runtime.managers.memory_managers.component_residency import (
- COMPONENT_OFFLOAD,
- LAYERWISE_OFFLOAD,
- RESIDENT,
-)
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload_components import (
is_dit_component_name,
)
-from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
-
-logger = init_logger(__name__)
GIB_BYTES = 1024**3
# Activation memory rarely scales perfectly linearly with workload units;
# pad the extrapolated activation part before checking the budget.
ACTIVATION_EXTRAPOLATION_MARGIN = 1.2
-# A target-shape measurement plus the mandatory post-placement warmup justifies
-# a tighter reserve than an extrapolated estimate. Both retain an absolute
-# floor for allocator slack, shape variance, and CUDA graph or compile pools.
-MEASURED_VRAM_RESERVE_FRACTION = 0.05
-EXTRAPOLATED_VRAM_RESERVE_FRACTION = 0.10
-MIN_VRAM_RESERVE_BYTES = 4 * GIB_BYTES
-# The absolute floor is sized for datacenter cards, where either fraction can
-# dominate. On a 12 GiB card a flat 4 GiB would fence off a third of the device,
-# so cap the floor as a share of what is actually there.
-MAX_VRAM_RESERVE_FRACTION = 0.20
-
-# A feasible placement is not automatically useful. Predictions inside this
-# interval are treated as latency-equivalent. The joint optimizer then avoids
-# changing strategy, minimizes device memory, preserves the faster estimate,
-# and finally minimizes HostPin. The raw estimate is already an upper bound:
-# transfer time is capped by the measured request.
-ESTIMATED_PINNED_H2D_BYTES_PER_SECOND = 24 * GIB_BYTES
-MIN_LATENCY_EQUIVALENCE_NS = 50_000_000
-MAX_LATENCY_EQUIVALENCE_NS = 100_000_000
-LATENCY_EQUIVALENCE_FRACTION = 0.01
-# The transfer model ranks feasible placements; the mandatory warmup is the
-# authority on whether a selected placement actually helped. Allow normal
-# measurement noise, but undo a round whose calibrated request is materially
-# slower than the original layout.
-POST_ADJUSTMENT_REGRESSION_FRACTION = 0.05
-
-PLACEMENT_STATUS_SKIPPED = "skipped"
-PLACEMENT_STATUS_ADJUSTED = "adjusted"
-PLACEMENT_STATUS_VALIDATED = "validated"
-PLACEMENT_STATUS_ROLLED_BACK = "rolled_back"
-PLACEMENT_STATUS_ROLLBACK_FAILED = "rollback_failed"
-
-
-def describe_error(error: BaseException) -> str:
- """Never-empty error text (str(AssertionError()) is "" and would be
- dropped by any truthiness filter)."""
- text = str(error)
- return f"{type(error).__name__}: {text}" if text else type(error).__name__
class WarmupMemoryRecord(msgspec.Struct, frozen=True):
@@ -113,96 +51,6 @@ class WarmupMemoryRecord(msgspec.Struct, frozen=True):
return max(1, self.width) * max(1, self.height) * max(1, self.num_frames)
-class ResidencyTarget(msgspec.Struct, frozen=True):
- """One complete target state for an auto-managed component."""
-
- component_name: str
- residency_mode: str
- target_resident_weight_bytes: int
- # Estimated per-request host-to-device traffic this target removes.
- h2d_bytes_per_request: int
- # Layerwise candidates jointly choose stage-scoped GPU residency and host
- # pinning. None is used by ordinary component placement.
- target_layerwise_resident_layers: tuple[int, ...] | None = None
- target_layerwise_pinned_layers: tuple[tuple[int, ...], ...] | None = None
- pinned_host_delta_bytes: int = 0
- host_unpin_scratch_bytes: int = 0
- host_pin_scratch_bytes: int = 0
- host_materialize_scratch_bytes: int = 0
- # Signed device-memory delta while applying the placement before the
- # validation warmup. Layerwise -> resident materializes every managed
- # layer immediately; a demotion can release those bytes first and fund a
- # later materialization in the same transaction.
- device_transition_delta_bytes: int = 0
- permanent_residency: bool = False
- # Device-memory delta relative to the measured placement. A component
- # already loaded for its own phase has a different delta from phases where
- # it is absent; keeping both avoids adding the same weights twice.
- active_device_delta_bytes: int = 0
- # Delta when the component is already present because of async prefetch,
- # but is not the semantic owner of this phase.
- present_device_delta_bytes: int = 0
- inactive_device_delta_bytes: int = 0
- # None preserves the historical derived target for hand-built callers:
- # partial layerwise targets remain layerwise, every other option is
- # resident. Generated complete-state frontiers set this explicitly.
- target_residency_mode: str | None = None
- current_placement: bool = False
- target_device_weight_bytes: int = 0
- target_pinned_host_bytes: int = 0
-
- def target_mode(self) -> str:
- if self.target_residency_mode is not None:
- return self.target_residency_mode
- if (
- self.target_layerwise_resident_layers is not None
- and not self.permanent_residency
- ):
- return LAYERWISE_OFFLOAD
- return RESIDENT
-
- def option_key(self) -> str:
- target_mode = self.target_mode()
- if target_mode == COMPONENT_OFFLOAD:
- return f"{self.component_name}:component-offload"
- if self.target_layerwise_resident_layers is None:
- return f"{self.component_name}:resident"
- layer_counts = ",".join(
- str(count) for count in self.target_layerwise_resident_layers
- )
- pinned = "|".join(
- ",".join(str(index) for index in indices) or "-"
- for indices in self.target_layerwise_pinned_layers or ()
- )
- permanence = "permanent" if self.permanent_residency else "stage"
- return f"{self.component_name}:{permanence}:layers={layer_counts}:pins={pinned}"
-
-
-class RankResidencyReport(msgspec.Struct, frozen=True):
- """One rank's inputs to the replica-wide placement decision."""
-
- rank: int
- budget_bytes: int
- estimated_peak_bytes: int | None
- target_workload_measured: bool = False
- observed_reserved_bytes: int = 0
- estimated_peak_bytes_by_phase: dict[str, int] = {}
- active_components_by_phase: dict[str, tuple[str, ...]] = {}
- used_components_by_phase: dict[str, tuple[str, ...]] = {}
- full_weight_transition_components_by_phase: dict[str, tuple[str, ...]] = {}
- current_device_weight_bytes_by_component: dict[str, int] = {}
- node_rank: int = 0
- pinned_host_bytes: int = 0
- host_pin_capacity_bytes: int = 0
- host_transition_headroom_bytes: int = 0
- device_transition_allocated_bytes: int = 0
- estimated_request_duration_ns: int = 0
- measured_request_duration_ns: int = 0
- candidate_latency_savings_ns: dict[str, int] = {}
- candidates: list[ResidencyTarget] = []
- skip_reason: str | None = None
-
-
def estimate_layerwise_layer_uses(
*,
records: Iterable[WarmupMemoryRecord],
diff --git a/python/sglang/multimodal_gen/runtime/managers/memory_managers/component_loading_order.py b/python/sglang/multimodal_gen/runtime/managers/memory_managers/component_loading_order.py
index 890eaf92e7..44481df9fd 100644
--- a/python/sglang/multimodal_gen/runtime/managers/memory_managers/component_loading_order.py
+++ b/python/sglang/multimodal_gen/runtime/managers/memory_managers/component_loading_order.py
@@ -12,6 +12,7 @@ import os
from dataclasses import dataclass
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload_components import (
+ component_base_name,
is_dit_component_name,
is_image_encoder_component_name,
is_text_encoder_component_name,
@@ -34,13 +35,6 @@ class ComponentLoadSpec:
_WEIGHT_FILE_SUFFIXES = (".bin", ".pt", ".pth")
-def _component_base_name(component_name: str) -> str:
- prefix, separator, suffix = component_name.rpartition("_")
- if separator and suffix.isdigit():
- return prefix
- return component_name
-
-
def _component_variant_priority(component_name: str) -> int:
_, separator, suffix = component_name.rpartition("_")
if separator and suffix.isdigit():
@@ -50,7 +44,7 @@ def _component_variant_priority(component_name: str) -> int:
def component_load_risk_rank(component_name: str) -> int:
"""Fallback type rank when checkpoint size cannot be inferred."""
- candidate_names = (component_name, _component_base_name(component_name))
+ candidate_names = (component_name, component_base_name(component_name))
if any(is_dit_component_name(name) for name in candidate_names):
return 0
if any(is_text_encoder_component_name(name) for name in candidate_names):
diff --git a/python/sglang/multimodal_gen/runtime/models/dits/causal_wanvideo.py b/python/sglang/multimodal_gen/runtime/models/dits/causal_wanvideo.py
index 7eb8e9b8f4..8e58f39c3d 100644
--- a/python/sglang/multimodal_gen/runtime/models/dits/causal_wanvideo.py
+++ b/python/sglang/multimodal_gen/runtime/models/dits/causal_wanvideo.py
@@ -98,7 +98,6 @@ class CausalWanSelfAttention(nn.Module):
self.sink_size = sink_size
self.qk_norm = qk_norm
self.eps = eps
- self.parallel_attention = parallel_attention
# Scaled dot product attention
self.attn = LocalAttention(
diff --git a/python/sglang/multimodal_gen/runtime/models/dits/cosmos3video.py b/python/sglang/multimodal_gen/runtime/models/dits/cosmos3video.py
index bcd45d75c3..183edf0637 100644
--- a/python/sglang/multimodal_gen/runtime/models/dits/cosmos3video.py
+++ b/python/sglang/multimodal_gen/runtime/models/dits/cosmos3video.py
@@ -375,7 +375,6 @@ class DomainAwareLinear(nn.Module):
super().__init__()
self.input_size = input_size
self.output_size = output_size
- self.num_domains = num_domains
self.fc = nn.Embedding(num_domains, output_size * input_size)
self.bias = nn.Embedding(num_domains, output_size)
nn.init.xavier_uniform_(
@@ -583,6 +582,27 @@ def _build_mlp(
# -----------------------------------------------------------------------------
+def _initialize_attention_projections(module, prefix, quant_config):
+ module.q_size = module.num_attention_heads * module.head_dim
+ module.kv_size = module.num_key_value_heads * module.head_dim
+ module.to_qkv = MergedColumnParallelLinear(
+ module.hidden_size,
+ [module.q_size, module.kv_size, module.kv_size],
+ bias=False,
+ gather_output=False,
+ quant_config=quant_config,
+ prefix=add_prefix("to_qkv", prefix),
+ )
+ module.to_out = RowParallelLinear(
+ module.q_size,
+ module.hidden_size,
+ bias=False,
+ input_is_parallel=True,
+ quant_config=quant_config,
+ prefix=add_prefix("to_out", prefix),
+ )
+
+
class Cosmos3CausalAttention(nn.Module):
"""Understanding pathway: causal self-attention on text tokens."""
@@ -618,24 +638,7 @@ class Cosmos3CausalAttention(nn.Module):
self.local_num_attention_heads = num_attention_heads // self.tp_size
self.local_num_key_value_heads = num_key_value_heads // self.tp_size
- self.q_size = num_attention_heads * head_dim
- self.kv_size = num_key_value_heads * head_dim
- self.to_qkv = MergedColumnParallelLinear(
- hidden_size,
- [self.q_size, self.kv_size, self.kv_size],
- bias=False,
- gather_output=False,
- quant_config=quant_config,
- prefix=add_prefix("to_qkv", prefix),
- )
- self.to_out = RowParallelLinear(
- num_attention_heads * head_dim,
- hidden_size,
- bias=False,
- input_is_parallel=True,
- quant_config=quant_config,
- prefix=add_prefix("to_out", prefix),
- )
+ _initialize_attention_projections(self, prefix, quant_config)
# Per-head QK norm (optional; some backbones omit it on text).
if qk_norm:
@@ -754,24 +757,7 @@ class Cosmos3CrossAttention(nn.Module):
self.local_num_attention_heads = num_attention_heads // self.tp_size
self.local_num_key_value_heads = num_key_value_heads // self.tp_size
- self.q_size = num_attention_heads * head_dim
- self.kv_size = num_key_value_heads * head_dim
- self.to_qkv = MergedColumnParallelLinear(
- hidden_size,
- [self.q_size, self.kv_size, self.kv_size],
- bias=False,
- gather_output=False,
- quant_config=quant_config,
- prefix=add_prefix("to_qkv", prefix),
- )
- self.to_out = RowParallelLinear(
- num_attention_heads * head_dim,
- hidden_size,
- bias=False,
- input_is_parallel=True,
- quant_config=quant_config,
- prefix=add_prefix("to_out", prefix),
- )
+ _initialize_attention_projections(self, prefix, quant_config)
self.norm_q = RMSNorm(head_dim, eps=1e-6)
self.norm_k = RMSNorm(head_dim, eps=1e-6)
diff --git a/python/sglang/multimodal_gen/runtime/models/dits/flux.py b/python/sglang/multimodal_gen/runtime/models/dits/flux.py
index a5c2e9d353..584960c576 100644
--- a/python/sglang/multimodal_gen/runtime/models/dits/flux.py
+++ b/python/sglang/multimodal_gen/runtime/models/dits/flux.py
@@ -523,7 +523,6 @@ class FluxAttention(torch.nn.Module, AttentionModuleMixin):
)
self.local_heads = divide(self.heads, self.tp_size)
self.added_kv_proj_dim = added_kv_proj_dim
- self.added_proj_bias = added_proj_bias
self.use_fused_qkv = isinstance(quant_config, NunchakuConfig)
self.use_fused_added_qkv = isinstance(quant_config, NunchakuConfig)
diff --git a/python/sglang/multimodal_gen/runtime/models/dits/flux_2.py b/python/sglang/multimodal_gen/runtime/models/dits/flux_2.py
index f839d78bef..3b24523df8 100644
--- a/python/sglang/multimodal_gen/runtime/models/dits/flux_2.py
+++ b/python/sglang/multimodal_gen/runtime/models/dits/flux_2.py
@@ -573,7 +573,6 @@ class Flux2Attention(torch.nn.Module, AttentionModuleMixin):
self.dropout = dropout
self.added_kv_proj_dim = added_kv_proj_dim
- self.added_proj_bias = added_proj_bias
# Packed NVFP4 checkpoints already serialize QKV together. ModelOpt
# FP8 exports separate Diffusers tensors, but the loader can merge
diff --git a/python/sglang/multimodal_gen/runtime/models/dits/hunyuanvideo.py b/python/sglang/multimodal_gen/runtime/models/dits/hunyuanvideo.py
index 9e064ccbf1..04288d6412 100644
--- a/python/sglang/multimodal_gen/runtime/models/dits/hunyuanvideo.py
+++ b/python/sglang/multimodal_gen/runtime/models/dits/hunyuanvideo.py
@@ -669,7 +669,6 @@ class HunyuanVideoTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixi
self.out_channels = (
config.in_channels if config.out_channels is None else config.out_channels
)
- self.unpatchify_channels = self.out_channels
self.guidance_embeds = config.guidance_embeds
self.rope_dim_list = list(config.rope_axes_dim)
self.rope_theta = config.rope_theta
diff --git a/python/sglang/multimodal_gen/runtime/models/dits/lingbot_world.py b/python/sglang/multimodal_gen/runtime/models/dits/lingbot_world.py
index 18eca688d6..2ec1730311 100644
--- a/python/sglang/multimodal_gen/runtime/models/dits/lingbot_world.py
+++ b/python/sglang/multimodal_gen/runtime/models/dits/lingbot_world.py
@@ -704,16 +704,9 @@ class LingBotWorldTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixi
width_local: int,
device: torch.device,
) -> tuple[torch.Tensor, torch.Tensor]:
- token_start = rank * local_len
- token_indices = torch.arange(
- token_start, token_start + local_len, device=device, dtype=torch.long
+ return self.rotary_emb.forward_3d_sequence_shard(
+ local_len, rank, frame_stride_local, width_local, device
)
- t_idx = token_indices // frame_stride_local
- rem = token_indices % frame_stride_local
- h_idx = rem // width_local
- w_idx = rem % width_local
- positions = torch.stack((t_idx, h_idx, w_idx), dim=1)
- return self.rotary_emb.forward_uncached(positions)
def forward(
self,
diff --git a/python/sglang/multimodal_gen/runtime/models/dits/ltx_2.py b/python/sglang/multimodal_gen/runtime/models/dits/ltx_2.py
index 18ab9df5f7..41715ba9ae 100644
--- a/python/sglang/multimodal_gen/runtime/models/dits/ltx_2.py
+++ b/python/sglang/multimodal_gen/runtime/models/dits/ltx_2.py
@@ -520,11 +520,6 @@ class LTX2AudioVideoRotaryPosEmbed(nn.Module):
audio_coords = audio_coords.unsqueeze(1)
return audio_coords
- def prepare_coords(self, *args, **kwargs):
- if self.modality == "video":
- return self.prepare_video_coords(*args, **kwargs)
- return self.prepare_audio_coords(*args, **kwargs)
-
def forward(
self,
coords: torch.Tensor,
@@ -697,7 +692,6 @@ class LTX2TPRMSNormAcrossHeads(nn.Module):
) -> None:
super().__init__()
self.full_hidden_size = full_hidden_size
- self.local_hidden_size = local_hidden_size
self.eps = eps
self.weight = nn.Parameter(torch.ones(local_hidden_size))
@@ -1864,8 +1858,6 @@ class LTX2VideoTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixin):
num_attention_heads=self.audio_num_attention_heads,
)
- self.cross_pe_max_pos = cross_attn_pos_embed_max_pos
-
# 5. Transformer Blocks
self.transformer_blocks = nn.ModuleList(
[
diff --git a/python/sglang/multimodal_gen/runtime/models/dits/mova_audio_dit.py b/python/sglang/multimodal_gen/runtime/models/dits/mova_audio_dit.py
index 0dc76d999f..1983b53001 100644
--- a/python/sglang/multimodal_gen/runtime/models/dits/mova_audio_dit.py
+++ b/python/sglang/multimodal_gen/runtime/models/dits/mova_audio_dit.py
@@ -15,7 +15,6 @@ from torch.distributed.tensor import DTensor
from sglang.multimodal_gen.configs.models.dits.mova_audio import MOVAAudioConfig
from sglang.multimodal_gen.configs.models.fsdp import is_block
from sglang.multimodal_gen.runtime.layers.linear import ReplicatedLinear
-from sglang.multimodal_gen.runtime.layers.mlp import MLP
from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config import (
QuantizationConfig,
)
@@ -25,7 +24,11 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload im
from sglang.multimodal_gen.runtime.models.dits.base import CachableDiT
# Reuse common functions and classes from mova_video_dit
-from .mova_video_dit import DiTBlock, precompute_freqs_cis, sinusoidal_embedding_1d
+from .mova_video_dit import (
+ initialize_mova_transformer_layers,
+ precompute_freqs_cis,
+ sinusoidal_embedding_1d,
+)
def precompute_freqs_cis_1d(dim: int, end: int = 16384, theta: float = 10000.0):
@@ -102,15 +105,11 @@ class WanAudioModel(CachableDiT, LayerwiseOffloadableModuleMixin):
# Extract parameters from config
dim = config.dim
in_dim = config.in_dim
- ffn_dim = config.ffn_dim
out_dim = config.out_dim
- text_dim = config.text_dim
freq_dim = config.freq_dim
eps = config.eps
patch_size = config.patch_size
num_heads = config.num_heads
- num_layers = config.num_layers
- has_image_pos_emb = config.has_image_pos_emb
has_ref_conv = config.has_ref_conv
separated_timestep = config.separated_timestep
require_vae_embedding = config.require_vae_embedding
@@ -126,38 +125,15 @@ class WanAudioModel(CachableDiT, LayerwiseOffloadableModuleMixin):
self.require_clip_embedding = require_clip_embedding
self.fuse_vae_embedding_in_latents = fuse_vae_embedding_in_latents
self.vae_type = vae_type
- # self.patch_embedding = nn.Conv3d(
- # in_dim, dim, kernel_size=patch_size, stride=patch_size)
self.patch_embedding = Conv1dLocalIsland(
in_dim, dim, kernel_size=patch_size, stride=patch_size
)
- self.text_embedding = MLP(
- text_dim,
- dim,
- output_dim=dim,
- act_type="gelu_pytorch_tanh",
- quant_config=quant_config,
- )
- self.time_embedding = MLP(
- freq_dim, dim, output_dim=dim, act_type="silu", quant_config=quant_config
- )
- # Preserve state_dict keys (time_projection.1.weight/bias).
- self.time_projection = nn.Sequential(
- nn.SiLU(), ReplicatedLinear(dim, dim * 6, quant_config=quant_config)
- )
- self.blocks = nn.ModuleList(
- [
- DiTBlock(dim, num_heads, ffn_dim, eps, quant_config=quant_config)
- for _ in range(num_layers)
- ]
- )
+ initialize_mova_transformer_layers(self, config, quant_config)
self.head = Head(dim, out_dim, patch_size, eps)
self.num_heads = num_heads
self.freqs = None
- self.img_pos_emb = None
if has_ref_conv:
self.ref_conv = nn.Conv2d(16, dim, kernel_size=(2, 2), stride=(2, 2))
- self.has_image_pos_emb = has_image_pos_emb
self.has_ref_conv = has_ref_conv
self.hidden_size = dim
self.num_attention_heads = num_heads
diff --git a/python/sglang/multimodal_gen/runtime/models/dits/mova_video_dit.py b/python/sglang/multimodal_gen/runtime/models/dits/mova_video_dit.py
index eff8494627..9f8572d3dc 100644
--- a/python/sglang/multimodal_gen/runtime/models/dits/mova_video_dit.py
+++ b/python/sglang/multimodal_gen/runtime/models/dits/mova_video_dit.py
@@ -272,9 +272,6 @@ class CrossAttention(_MOVAAttention):
class MulAdd(nn.Module):
- def __init__(self):
- super().__init__()
-
def forward(self, x, gate, residual):
return residual + gate * x
@@ -408,6 +405,41 @@ class Conv3dLocalIsland(nn.Conv3d):
return super().forward(input)
+def initialize_mova_transformer_layers(model, config, quant_config):
+ """Create shared audio/video layers in checkpoint registration order."""
+ model.text_embedding = MLP(
+ config.text_dim,
+ config.dim,
+ output_dim=config.dim,
+ act_type="gelu_pytorch_tanh",
+ quant_config=quant_config,
+ )
+ model.time_embedding = MLP(
+ config.freq_dim,
+ config.dim,
+ output_dim=config.dim,
+ act_type="silu",
+ quant_config=quant_config,
+ )
+ # Preserve state_dict keys (time_projection.1.weight/bias).
+ model.time_projection = nn.Sequential(
+ nn.SiLU(),
+ ReplicatedLinear(config.dim, config.dim * 6, quant_config=quant_config),
+ )
+ model.blocks = nn.ModuleList(
+ [
+ DiTBlock(
+ config.dim,
+ config.num_heads,
+ config.ffn_dim,
+ config.eps,
+ quant_config=quant_config,
+ )
+ for _ in range(config.num_layers)
+ ]
+ )
+
+
class WanModel(CachableDiT, LayerwiseOffloadableModuleMixin):
_fsdp_shard_conditions = [is_block]
_compile_conditions = [is_block]
@@ -426,15 +458,11 @@ class WanModel(CachableDiT, LayerwiseOffloadableModuleMixin):
# Extract parameters from config
dim = config.dim
in_dim = config.in_dim
- ffn_dim = config.ffn_dim
out_dim = config.out_dim
- text_dim = config.text_dim
freq_dim = config.freq_dim
eps = config.eps
patch_size = config.patch_size
num_heads = config.num_heads
- num_layers = config.num_layers
- has_image_pos_emb = config.has_image_pos_emb
has_ref_conv = config.has_ref_conv
separated_timestep = config.separated_timestep
require_vae_embedding = config.require_vae_embedding
@@ -452,33 +480,13 @@ class WanModel(CachableDiT, LayerwiseOffloadableModuleMixin):
self.patch_embedding = Conv3dLocalIsland(
in_dim, dim, kernel_size=patch_size, stride=patch_size
)
- self.text_embedding = MLP(
- text_dim,
- dim,
- output_dim=dim,
- act_type="gelu_pytorch_tanh",
- quant_config=quant_config,
- )
- self.time_embedding = MLP(
- freq_dim, dim, output_dim=dim, act_type="silu", quant_config=quant_config
- )
- # Preserve state_dict keys (time_projection.1.weight/bias).
- self.time_projection = nn.Sequential(
- nn.SiLU(), ReplicatedLinear(dim, dim * 6, quant_config=quant_config)
- )
- self.blocks = nn.ModuleList(
- [
- DiTBlock(dim, num_heads, ffn_dim, eps, quant_config=quant_config)
- for _ in range(num_layers)
- ]
- )
+ initialize_mova_transformer_layers(self, config, quant_config)
self.head = Head(dim, out_dim, patch_size, eps)
self.num_heads = num_heads
self.freqs = None
if has_ref_conv:
self.ref_conv = nn.Conv2d(16, dim, kernel_size=(2, 2), stride=(2, 2))
- self.has_image_pos_emb = has_image_pos_emb
self.has_ref_conv = has_ref_conv
self.hidden_size = dim
self.num_attention_heads = num_heads
diff --git a/python/sglang/multimodal_gen/runtime/models/dits/qwen_image.py b/python/sglang/multimodal_gen/runtime/models/dits/qwen_image.py
index 71c9bd2542..f906c6ab39 100644
--- a/python/sglang/multimodal_gen/runtime/models/dits/qwen_image.py
+++ b/python/sglang/multimodal_gen/runtime/models/dits/qwen_image.py
@@ -744,7 +744,6 @@ class QwenImageCrossAttention(nn.Module):
self.window_size = window_size
self.qk_norm = qk_norm
self.eps = eps
- self.parallel_attention = parallel_attention
self.added_kv_proj_dim = added_kv_proj_dim
self.prefix = prefix
self.defer_output_bias = _defer_modelopt_output_bias(quant_config)
diff --git a/python/sglang/multimodal_gen/runtime/models/dits/sana_wm.py b/python/sglang/multimodal_gen/runtime/models/dits/sana_wm.py
index 2365fe91f1..aac01d8ef0 100644
--- a/python/sglang/multimodal_gen/runtime/models/dits/sana_wm.py
+++ b/python/sglang/multimodal_gen/runtime/models/dits/sana_wm.py
@@ -169,6 +169,17 @@ class SanaWMBlock(nn.Module):
tokens_per_frame = N // num_frames
return x.reshape(B, num_frames, tokens_per_frame, C), tokens_per_frame
+ def _prepare_modulation(self, t: torch.Tensor, batch_size: int):
+ if t.dim() == 2:
+ return None, (
+ self.scale_shift_table[None] + t.reshape(batch_size, 6, -1)
+ ).chunk(6, dim=1)
+ num_frames = t.reshape(batch_size, -1, 6, t.shape[-1] // 6).shape[1]
+ t = t.reshape(batch_size, num_frames, 6, -1)
+ return num_frames, (self.scale_shift_table[None, None, :, :] + t).chunk(
+ 6, dim=2
+ )
+
def forward(
self,
x: torch.Tensor, # (B, N, D)
@@ -184,17 +195,8 @@ class SanaWMBlock(nn.Module):
chunk_index: Optional[List[int]] = None,
) -> torch.Tensor:
B = x.shape[0]
- if t.dim() == 2:
- num_frames = None
- shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = (
- self.scale_shift_table[None] + t.reshape(B, 6, -1)
- ).chunk(6, dim=1)
- else:
- num_frames = t.reshape(B, -1, 6, t.shape[-1] // 6).shape[1]
- t = t.reshape(B, num_frames, 6, -1)
- shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = (
- self.scale_shift_table[None, None, :, :] + t
- ).chunk(6, dim=2)
+ num_frames, modulation = self._prepare_modulation(t, B)
+ shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = modulation
# Self-attention with UCPE camera branch
if num_frames is None:
@@ -259,17 +261,8 @@ class SanaWMBlock(nn.Module):
) -> Tuple[torch.Tensor, list]:
"""Streaming counterpart of ``forward``: threads the per-block 10-slot ``kv_cache`` through cached attention + FFN."""
B = x.shape[0]
- if t.dim() == 2:
- num_frames = None
- shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = (
- self.scale_shift_table[None] + t.reshape(B, 6, -1)
- ).chunk(6, dim=1)
- else:
- num_frames = t.reshape(B, -1, 6, t.shape[-1] // 6).shape[1]
- t = t.reshape(B, num_frames, 6, -1)
- shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = (
- self.scale_shift_table[None, None, :, :] + t
- ).chunk(6, dim=2)
+ num_frames, modulation = self._prepare_modulation(t, B)
+ shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = modulation
if num_frames is None:
x_in = self._modulate(self.norm1(x), shift_msa, scale_msa)
@@ -435,7 +428,6 @@ class SanaWMTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixin):
for i in range(depth)
if arch.softmax_every_n > 0 and (i + 1) % arch.softmax_every_n == 0
)
- self.softmax_block_indices = tuple(sorted(softmax_idx))
self.blocks = nn.ModuleList(
[
diff --git a/python/sglang/multimodal_gen/runtime/models/dits/wanvideo.py b/python/sglang/multimodal_gen/runtime/models/dits/wanvideo.py
index ec83bfc922..66e40dc2c8 100755
--- a/python/sglang/multimodal_gen/runtime/models/dits/wanvideo.py
+++ b/python/sglang/multimodal_gen/runtime/models/dits/wanvideo.py
@@ -203,7 +203,6 @@ class WanSelfAttention(nn.Module):
self.window_size = window_size
self.qk_norm = qk_norm
self.eps = eps
- self.parallel_attention = parallel_attention
tp_size = get_tp_world_size()
# layers
@@ -1137,19 +1136,9 @@ class WanTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixin):
width_local: int,
device: torch.device,
) -> tuple[torch.Tensor, torch.Tensor]:
- token_start = rank * local_len
- token_indices = torch.arange(
- token_start,
- token_start + local_len,
- device=device,
- dtype=torch.long,
+ return self.rotary_emb.forward_3d_sequence_shard(
+ local_len, rank, frame_stride_local, width_local, device
)
- t_idx = token_indices // frame_stride_local
- rem = token_indices % frame_stride_local
- h_idx = rem // width_local
- w_idx = rem % width_local
- positions = torch.stack((t_idx, h_idx, w_idx), dim=1)
- return self.rotary_emb.forward_uncached(positions)
def forward(
self,
diff --git a/python/sglang/multimodal_gen/runtime/models/dits/zimage.py b/python/sglang/multimodal_gen/runtime/models/dits/zimage.py
index d976e6e141..8226062724 100644
--- a/python/sglang/multimodal_gen/runtime/models/dits/zimage.py
+++ b/python/sglang/multimodal_gen/runtime/models/dits/zimage.py
@@ -1379,10 +1379,6 @@ class ZImageTransformer2DModel(CachableDiT, LayerwiseOffloadableModuleMixin):
self._cached_joint_attn_mask_meta = (cache_key, result)
return self._pin_for_active_capture(result)
- @staticmethod
- def _has_padding(valid_lens: list[int], target_len: int) -> bool:
- return any(int(length) < target_len for length in valid_lens)
-
@staticmethod
def _as_image_list(hidden_states) -> list[torch.Tensor]:
"""Normalize 4D/5D image latents into per-sample tensors."""
diff --git a/python/sglang/multimodal_gen/runtime/models/encoders/base.py b/python/sglang/multimodal_gen/runtime/models/encoders/base.py
index 21eaae1905..c191c9a6d8 100644
--- a/python/sglang/multimodal_gen/runtime/models/encoders/base.py
+++ b/python/sglang/multimodal_gen/runtime/models/encoders/base.py
@@ -31,6 +31,18 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload im
from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
+def get_attention_head_partition(
+ num_heads: int, num_kv_heads: int, tp_size: int
+) -> tuple[int, int]:
+ """Partition query heads and shard or replicate KV heads across TP ranks."""
+ assert num_heads % tp_size == 0
+ if num_kv_heads >= tp_size:
+ assert num_kv_heads % tp_size == 0
+ else:
+ assert tp_size % num_kv_heads == 0
+ return num_heads // tp_size, max(1, num_kv_heads // tp_size)
+
+
def get_folding_tp_group(config: EncoderConfig):
"""Return the TP group selected for an encoder."""
mode = config.parallel_folding_mode
@@ -218,7 +230,6 @@ class TextEncoder(
# shard conditions left in the root group stays sharded unless the entry
# point is registered; loaders read this and register each name.
_fsdp_forward_methods: tuple[str, ...] = ()
- _stacked_params_mapping: list[tuple[str, str, str]] = field(default_factory=list)
_supported_attention_backends: set[AttentionBackendEnum] = (
TextEncoderConfig()._supported_attention_backends
)
@@ -227,7 +238,6 @@ class TextEncoder(
super().__init__()
self.config = config
self._fsdp_shard_conditions = config.arch_config._fsdp_shard_conditions
- self._stacked_params_mapping = config.arch_config.stacked_params_mapping
if not self.supported_attention_backends:
raise ValueError(
f"Subclass {self.__class__.__name__} must define _supported_attention_backends"
diff --git a/python/sglang/multimodal_gen/runtime/models/encoders/gemma2.py b/python/sglang/multimodal_gen/runtime/models/encoders/gemma2.py
index 7edcc82a47..0511c9868f 100644
--- a/python/sglang/multimodal_gen/runtime/models/encoders/gemma2.py
+++ b/python/sglang/multimodal_gen/runtime/models/encoders/gemma2.py
@@ -40,6 +40,7 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload im
)
from sglang.multimodal_gen.runtime.models.encoders.base import (
EncoderTensorParallelMixin,
+ get_attention_head_partition,
)
logger = logging.getLogger(__name__)
@@ -114,14 +115,10 @@ class Gemma2Attention(nn.Module):
self.hidden_size = hidden_size
tp_size = get_tp_world_size()
self.total_num_heads = num_heads
- assert self.total_num_heads % tp_size == 0
- self.num_heads = self.total_num_heads // tp_size
self.total_num_kv_heads = num_kv_heads
- if self.total_num_kv_heads >= tp_size:
- assert self.total_num_kv_heads % tp_size == 0
- else:
- assert tp_size % self.total_num_kv_heads == 0
- self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
+ self.num_heads, self.num_kv_heads = get_attention_head_partition(
+ num_heads, num_kv_heads, tp_size
+ )
arch = config.arch_config
self.head_dim = arch.head_dim
diff --git a/python/sglang/multimodal_gen/runtime/models/encoders/gemma_3.py b/python/sglang/multimodal_gen/runtime/models/encoders/gemma_3.py
index 5dc62b7c69..a4d4c3edf1 100644
--- a/python/sglang/multimodal_gen/runtime/models/encoders/gemma_3.py
+++ b/python/sglang/multimodal_gen/runtime/models/encoders/gemma_3.py
@@ -27,6 +27,7 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload im
)
from sglang.multimodal_gen.runtime.models.encoders.base import (
EncoderTensorParallelMixin,
+ get_attention_head_partition,
)
from sglang.multimodal_gen.runtime.utils.common import add_prefix
from sglang.srt.models.siglip import SiglipVisionModel
@@ -68,6 +69,36 @@ def get_attention_sliding_window_size(config):
return config.sliding_window - 1
+def _load_with_shard_id(
+ weight_loader, param, loaded_weight: torch.Tensor, shard_id
+) -> None:
+ """Try the supplied shard ID, then its equivalent q/k/v or integer form."""
+ try:
+ weight_loader(param, loaded_weight, shard_id)
+ return
+ except (AssertionError, TypeError):
+ pass
+
+ if isinstance(shard_id, str):
+ mapping = {"q": 0, "k": 1, "v": 2}
+ if shard_id in mapping:
+ weight_loader(param, loaded_weight, mapping[shard_id])
+ return
+ if shard_id.isdigit():
+ weight_loader(param, loaded_weight, int(shard_id))
+ return
+ elif isinstance(shard_id, int):
+ mapping = {0: "q", 1: "k", 2: "v"}
+ if shard_id in mapping:
+ weight_loader(param, loaded_weight, mapping[shard_id])
+ return
+
+ raise TypeError(
+ f"Unsupported shard_id={shard_id!r} for weight_loader={weight_loader} "
+ f"(param={getattr(param, 'name', '')})."
+ )
+
+
class Gemma3RMSNorm(nn.Module):
def __init__(self, dim: int, eps: float = 1e-6):
super().__init__()
@@ -146,14 +177,10 @@ class Gemma3Attention(nn.Module):
self.hidden_size = hidden_size
tp_size = get_tp_world_size()
self.total_num_heads = num_heads
- assert self.total_num_heads % tp_size == 0
- self.num_heads = self.total_num_heads // tp_size
self.total_num_kv_heads = num_kv_heads
- if self.total_num_kv_heads >= tp_size:
- assert self.total_num_kv_heads % tp_size == 0
- else:
- assert tp_size % self.total_num_kv_heads == 0
- self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
+ self.num_heads, self.num_kv_heads = get_attention_head_partition(
+ num_heads, num_kv_heads, tp_size
+ )
self.head_dim = getattr(
config.text_config, "head_dim", self.hidden_size // self.total_num_heads
diff --git a/python/sglang/multimodal_gen/runtime/models/encoders/llama.py b/python/sglang/multimodal_gen/runtime/models/encoders/llama.py
index 2b8190cb25..76d927fb20 100644
--- a/python/sglang/multimodal_gen/runtime/models/encoders/llama.py
+++ b/python/sglang/multimodal_gen/runtime/models/encoders/llama.py
@@ -51,10 +51,12 @@ from sglang.multimodal_gen.runtime.layers.vocab_parallel_embedding import (
VocabParallelEmbedding,
)
from sglang.multimodal_gen.runtime.loader.weight_utils import (
- load_stacked_weight,
- maybe_remap_kv_scale_name,
+ load_llm_encoder_weights,
+)
+from sglang.multimodal_gen.runtime.models.encoders.base import (
+ TextEncoder,
+ get_attention_head_partition,
)
-from sglang.multimodal_gen.runtime.models.encoders.base import TextEncoder
class LlamaMLP(nn.Module):
@@ -116,18 +118,10 @@ class LlamaAttention(nn.Module):
self.hidden_size = hidden_size
tp_size = get_tp_world_size()
self.total_num_heads = num_heads
- assert self.total_num_heads % tp_size == 0
- self.num_heads = self.total_num_heads // tp_size
self.total_num_kv_heads = num_kv_heads
- if self.total_num_kv_heads >= tp_size:
- # Number of KV heads is greater than TP size, so we partition
- # the KV heads across multiple tensor parallel GPUs.
- assert self.total_num_kv_heads % tp_size == 0
- else:
- # Number of KV heads is less than TP size, so we replicate
- # the KV heads across multiple tensor parallel GPUs.
- assert tp_size % self.total_num_kv_heads == 0
- self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
+ self.num_heads, self.num_kv_heads = get_attention_head_partition(
+ num_heads, num_kv_heads, tp_size
+ )
# MistralConfig has an optional head_dim introduced by Mistral-Nemo
self.head_dim = getattr(
config, "head_dim", self.hidden_size // self.total_num_heads
@@ -390,32 +384,11 @@ class LlamaModel(TextEncoder):
return output
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
-
- params_dict = dict(self.named_parameters())
- loaded_params: set[str] = set()
- for name, loaded_weight in weights:
- if "rotary_emb.inv_freq" in name:
- continue
- if "rotary_emb.cos_cached" in name or "rotary_emb.sin_cached" in name:
- # Models trained using ColossalAI may include these tensors in
- # the checkpoint. Skip them.
- continue
- if "scale" in name:
- # Remapping the name of FP8 kv-scale.
- kv_scale_name: str | None = maybe_remap_kv_scale_name(name, params_dict)
- if kv_scale_name is None:
- continue
- else:
- name = kv_scale_name
- name = load_stacked_weight(
- name,
- loaded_weight,
- params_dict,
- self.config.arch_config.stacked_params_mapping,
- )
- if name is not None:
- loaded_params.add(name)
- return loaded_params
+ return load_llm_encoder_weights(
+ weights,
+ dict(self.named_parameters()),
+ self.config.arch_config.stacked_params_mapping,
+ )
EntryClass = LlamaModel
diff --git a/python/sglang/multimodal_gen/runtime/models/encoders/qwen2_5vl.py b/python/sglang/multimodal_gen/runtime/models/encoders/qwen2_5vl.py
index 1db47cbfe1..130219d475 100644
--- a/python/sglang/multimodal_gen/runtime/models/encoders/qwen2_5vl.py
+++ b/python/sglang/multimodal_gen/runtime/models/encoders/qwen2_5vl.py
@@ -1332,13 +1332,10 @@ class Qwen2_5_VLForConditionalGeneration(TextEncoder):
break
if loaded_stacked_param:
continue
- try:
- # Skip loading extra bias for GPTQ models.
- if name.endswith(".bias") and name not in params_dict:
- continue
- param = params_dict[name]
- except KeyError:
- raise
+ # Skip loading extra bias for GPTQ models.
+ if name.endswith(".bias") and name not in params_dict:
+ continue
+ param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
loaded_weight = loaded_weight.to(param.dtype)
diff --git a/python/sglang/multimodal_gen/runtime/models/encoders/qwen3.py b/python/sglang/multimodal_gen/runtime/models/encoders/qwen3.py
index b74d93c7fb..1a9f4d6c05 100644
--- a/python/sglang/multimodal_gen/runtime/models/encoders/qwen3.py
+++ b/python/sglang/multimodal_gen/runtime/models/encoders/qwen3.py
@@ -20,10 +20,12 @@ from sglang.multimodal_gen.runtime.layers.vocab_parallel_embedding import (
VocabParallelEmbedding,
)
from sglang.multimodal_gen.runtime.loader.weight_utils import (
- load_stacked_weight,
- maybe_remap_kv_scale_name,
+ load_llm_encoder_weights,
+)
+from sglang.multimodal_gen.runtime.models.encoders.base import (
+ TextEncoder,
+ get_attention_head_partition,
)
-from sglang.multimodal_gen.runtime.models.encoders.base import TextEncoder
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.layernorm import RMSNorm
@@ -91,14 +93,10 @@ class Qwen3Attention(nn.Module):
self.hidden_size = hidden_size
tp_size = get_tp_world_size()
self.total_num_heads = num_heads
- assert self.total_num_heads % tp_size == 0
- self.num_heads = self.total_num_heads // tp_size
self.total_num_kv_heads = num_kv_heads
- if self.total_num_kv_heads >= tp_size:
- assert self.total_num_kv_heads % tp_size == 0
- else:
- assert tp_size % self.total_num_kv_heads == 0
- self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
+ self.num_heads, self.num_kv_heads = get_attention_head_partition(
+ num_heads, num_kv_heads, tp_size
+ )
self.head_dim = getattr(
config, "head_dim", self.hidden_size // self.total_num_heads
@@ -432,38 +430,12 @@ class Qwen3ForCausalLM(TextEncoder):
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
"""Load weights with support for tensor parallelism and weight remapping."""
- params_dict = dict(self.named_parameters())
- loaded_params: set[str] = set()
-
- for name, loaded_weight in weights:
- # Strip 'model.' prefix from HuggingFace Qwen3 weights
- if name.startswith("model."):
- name = name[6:] # len("model.") == 6
-
- # Skip rotary embedding weights
- if "rotary_emb.inv_freq" in name:
- continue
- if "rotary_emb.cos_cached" in name or "rotary_emb.sin_cached" in name:
- continue
-
- # Handle KV scale remapping
- if "scale" in name:
- kv_scale_name: str | None = maybe_remap_kv_scale_name(name, params_dict)
- if kv_scale_name is None:
- continue
- else:
- name = kv_scale_name
-
- name = load_stacked_weight(
- name,
- loaded_weight,
- params_dict,
- self.config.arch_config.stacked_params_mapping,
- )
- if name is not None:
- loaded_params.add(name)
-
- return loaded_params
+ return load_llm_encoder_weights(
+ weights,
+ dict(self.named_parameters()),
+ self.config.arch_config.stacked_params_mapping,
+ strip_prefix="model.",
+ )
EntryClass = Qwen3ForCausalLM
diff --git a/python/sglang/multimodal_gen/runtime/models/encoders/t5.py b/python/sglang/multimodal_gen/runtime/models/encoders/t5.py
index 566a4ddbd3..333a152e6a 100644
--- a/python/sglang/multimodal_gen/runtime/models/encoders/t5.py
+++ b/python/sglang/multimodal_gen/runtime/models/encoders/t5.py
@@ -158,9 +158,6 @@ class T5LayerFF(nn.Module):
# T5 has attn_bias and does not use softmax scaling
class T5MultiHeadAttention(nn.Module):
- def __init__(self) -> None:
- super().__init__()
-
def forward(self, q, k, v, attn_bias=None):
b, _, n, c = q.shape
attn = torch.einsum("binc,bjnc->bnij", q, k)
@@ -509,7 +506,6 @@ class T5Stack(nn.Module):
):
super().__init__()
self.embed_tokens = embed_tokens
- self.is_umt5 = is_umt5
if is_umt5:
self.block = nn.ModuleList(
[
diff --git a/python/sglang/multimodal_gen/runtime/models/schedulers/flow_match_pair.py b/python/sglang/multimodal_gen/runtime/models/schedulers/flow_match_pair.py
index cb26f9fd9a..bfd9a8d985 100644
--- a/python/sglang/multimodal_gen/runtime/models/schedulers/flow_match_pair.py
+++ b/python/sglang/multimodal_gen/runtime/models/schedulers/flow_match_pair.py
@@ -49,7 +49,6 @@ class FlowMatchScheduler(BaseScheduler):
self,
num_inference_steps=100,
denoising_strength=1.0,
- training=False,
shift=None,
dynamic_shift_len=None,
):
@@ -91,17 +90,6 @@ class FlowMatchScheduler(BaseScheduler):
if self.train_timesteps is None:
self.train_timesteps = self.timesteps
self.train_sigmas = self.sigmas
- if training:
- x = self.timesteps
- y = torch.exp(
- -2 * ((x - num_inference_steps / 2) / num_inference_steps) ** 2
- )
- y_shifted = y - y.min()
- bsmntw_weighing = y_shifted * (num_inference_steps / y_shifted.sum())
- self.linear_timesteps_weights = bsmntw_weighing
- self.training = True
- else:
- self.training = False
def scale_model_input(self, sample: torch.Tensor, timestep: int | None = None):
return sample
@@ -118,14 +106,6 @@ class FlowMatchScheduler(BaseScheduler):
prev_sample = sample + model_output * (sigma_ - sigma)
return prev_sample
- def return_to_timestep(self, timestep, sample, sample_stablized):
- if isinstance(timestep, torch.Tensor):
- timestep = timestep.cpu()
- timestep_id = torch.argmin((self.timesteps - timestep).abs())
- sigma = self.sigmas[timestep_id]
- model_output = (sample - sample_stablized) / sigma
- return model_output
-
def add_noise(self, original_samples, noise, timestep):
if isinstance(timestep, torch.Tensor):
timestep = timestep.cpu()
@@ -134,17 +114,6 @@ class FlowMatchScheduler(BaseScheduler):
sample = (1 - sigma) * original_samples + sigma * noise
return sample
- def training_target(self, sample, noise, timestep):
- target = noise - sample
- return target
-
- def training_weight(self, timestep):
- timestep_id = torch.argmin(
- (self.timesteps - timestep.to(self.timesteps.device)).abs()
- )
- weights = self.linear_timesteps_weights[timestep_id]
- return weights
-
def calculate_shift(
self,
image_seq_len,
@@ -268,9 +237,9 @@ class FlowMatchPairScheduler(FlowMatchScheduler):
self.set_pair_postprocess(_quadratic_perp_bulge_swap)
return
- if name == "v2a_sequential":
+ if name in ("v2a_sequential", "a2v_sequential"):
- def _v2a(pairs: torch.Tensor):
+ def _sequential(pairs: torch.Tensor):
if (
not isinstance(pairs, torch.Tensor)
or pairs.ndim != 2
@@ -281,8 +250,12 @@ class FlowMatchPairScheduler(FlowMatchScheduler):
base = pairs[:, 0]
seq_half = base[::2]
m = int(seq_half.shape[0])
- col0 = torch.cat([seq_half, seq_half[-1:].repeat(m)], dim=0)[:N]
- col1 = torch.cat([seq_half[0:1].repeat(m), seq_half], dim=0)[:N]
+ if name == "v2a_sequential":
+ col0 = torch.cat([seq_half, seq_half[-1:].repeat(m)], dim=0)[:N]
+ col1 = torch.cat([seq_half[0:1].repeat(m), seq_half], dim=0)[:N]
+ else:
+ col0 = torch.cat([seq_half[0:1].repeat(m), seq_half], dim=0)[:N]
+ col1 = torch.cat([seq_half, seq_half[-1:].repeat(m)], dim=0)[:N]
return torch.stack(
[
col0.to(dtype=pairs.dtype, device=pairs.device),
@@ -291,60 +264,25 @@ class FlowMatchPairScheduler(FlowMatchScheduler):
dim=1,
)
- self.set_pair_postprocess(_v2a)
+ self.set_pair_postprocess(_sequential)
return
- if name == "a2v_sequential":
+ if name in ("v2a", "a2v"):
- def _a2v(pairs: torch.Tensor):
+ def _single_modality(pairs: torch.Tensor):
if (
not isinstance(pairs, torch.Tensor)
or pairs.ndim != 2
or pairs.shape[1] != 2
):
raise ValueError("pairs must be a torch.Tensor of shape [N, 2]")
- N = pairs.shape[0]
- base = pairs[:, 0]
- seq_half = base[::2]
- m = int(seq_half.shape[0])
- col0 = torch.cat([seq_half[0:1].repeat(m), seq_half], dim=0)[:N]
- col1 = torch.cat([seq_half, seq_half[-1:].repeat(m)], dim=0)[:N]
- return torch.stack(
- [
- col0.to(dtype=pairs.dtype, device=pairs.device),
- col1.to(dtype=pairs.dtype, device=pairs.device),
- ],
- dim=1,
- )
+ if name == "v2a":
+ zeros = torch.zeros_like(pairs[:, 0])
+ return torch.stack([zeros, pairs[:, 1]], dim=1)
+ else:
+ zeros = torch.zeros_like(pairs[:, 1])
+ return torch.stack([pairs[:, 0], zeros], dim=1)
- self.set_pair_postprocess(_a2v)
- return
- if name == "v2a":
-
- def _v2a_classic(pairs: torch.Tensor):
- if (
- not isinstance(pairs, torch.Tensor)
- or pairs.ndim != 2
- or pairs.shape[1] != 2
- ):
- raise ValueError("pairs must be a torch.Tensor of shape [N, 2]")
- zeros = torch.zeros_like(pairs[:, 0])
- return torch.stack([zeros, pairs[:, 1]], dim=1)
-
- self.set_pair_postprocess(_v2a_classic)
- return
- if name == "a2v":
-
- def _a2v_classic(pairs: torch.Tensor):
- if (
- not isinstance(pairs, torch.Tensor)
- or pairs.ndim != 2
- or pairs.shape[1] != 2
- ):
- raise ValueError("pairs must be a torch.Tensor of shape [N, 2]")
- zeros = torch.zeros_like(pairs[:, 1])
- return torch.stack([pairs[:, 0], zeros], dim=1)
-
- self.set_pair_postprocess(_a2v_classic)
+ self.set_pair_postprocess(_single_modality)
return
if name == "dual_sigma_shift":
visual_shift = float(kwargs.get("visual_shift", self.shift))
diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/alias_free.py b/python/sglang/multimodal_gen/runtime/models/vaes/alias_free.py
index b45405c1a2..16f1085b47 100644
--- a/python/sglang/multimodal_gen/runtime/models/vaes/alias_free.py
+++ b/python/sglang/multimodal_gen/runtime/models/vaes/alias_free.py
@@ -19,21 +19,7 @@ import torch
import torch.nn as nn
import torch.nn.functional as F
-if "sinc" in dir(torch):
- sinc = torch.sinc
-else:
- # This code is adopted from adefossez's julius.core.sinc under the MIT License
- # https://adefossez.github.io/julius/julius/core.html
- def sinc(x: torch.Tensor):
- """
- Implementation of sinc, i.e. sin(pi * x) / (pi * x)
- __Warning__: Different to julius.sinc, the input is multiplied by `pi`!
- """
- return torch.where(
- x == 0,
- torch.tensor(1.0, device=x.device, dtype=x.dtype),
- torch.sin(math.pi * x) / math.pi / x,
- )
+sinc = torch.sinc
# This code is adopted from adefossez's julius.lowpass.LowPassFilters under the MIT License
@@ -175,8 +161,6 @@ class Activation1d(nn.Module):
down_kernel_size: int = 12,
):
super().__init__()
- self.up_ratio = up_ratio
- self.down_ratio = down_ratio
self.act = activation
self.upsample = UpSample1d(up_ratio, up_kernel_size)
self.downsample = DownSample1d(down_ratio, down_kernel_size)
diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_kl_qwenimage.py b/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_kl_qwenimage.py
index 9bf4d4eeaa..3a729a22cc 100644
--- a/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_kl_qwenimage.py
+++ b/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_kl_qwenimage.py
@@ -951,18 +951,6 @@ class AutoencoderKLQwenImage(ParallelTiledVAE):
self.tile_sample_stride_height = 192
self.tile_sample_stride_width = 192
- # Precompute and cache conv counts for encoder and decoder for clear_cache speedup
- self._cached_conv_counts = {
- "decoder": sum(
- isinstance(m, (QwenImageCausalConv3d, SpatialParallelCausalConv3d))
- for m in self.decoder.modules()
- )
- if self.decoder is not None
- else 0,
- "encoder": sum(isinstance(m, QwenImageCausalConv3d) for m in self.encoder.modules())
- if self.encoder is not None
- else 0,
- }
cuda_device = get_local_torch_device()
dtype = torch.get_default_dtype()
latent_channels = config.arch_config.z_dim
diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/common.py b/python/sglang/multimodal_gen/runtime/models/vaes/common.py
index ab613cbcf2..29ceb25222 100644
--- a/python/sglang/multimodal_gen/runtime/models/vaes/common.py
+++ b/python/sglang/multimodal_gen/runtime/models/vaes/common.py
@@ -7,7 +7,6 @@ from functools import lru_cache
from math import isqrt, prod
from typing import Optional, cast
-import numpy as np
import torch
import torch.distributed as dist
from diffusers.models.autoencoders.vae import DiagonalGaussianDistribution
@@ -230,6 +229,14 @@ class ParallelTiledVAE(ABC, nn.Module, LayerwiseOffloadableModuleMixin):
) + b[:, :, x, :, :] * (x / blend_extent)
return b
+ def _spatial_tile_geometry(self) -> tuple[int, int, int, int]:
+ return (
+ self.tile_sample_min_height // self.spatial_compression_ratio,
+ self.tile_sample_min_width // self.spatial_compression_ratio,
+ self.tile_sample_stride_height // self.spatial_compression_ratio,
+ self.tile_sample_stride_width // self.spatial_compression_ratio,
+ )
+
def spatial_tiled_encode(self, x: torch.Tensor) -> torch.Tensor:
r"""Encode a batch of images using a tiled encoder.
@@ -244,18 +251,12 @@ class ParallelTiledVAE(ABC, nn.Module, LayerwiseOffloadableModuleMixin):
# latent_height = height // self.spatial_compression_ratio
# latent_width = width // self.spatial_compression_ratio
- tile_latent_min_height = (
- self.tile_sample_min_height // self.spatial_compression_ratio
- )
- tile_latent_min_width = (
- self.tile_sample_min_width // self.spatial_compression_ratio
- )
- tile_latent_stride_height = (
- self.tile_sample_stride_height // self.spatial_compression_ratio
- )
- tile_latent_stride_width = (
- self.tile_sample_stride_width // self.spatial_compression_ratio
- )
+ (
+ tile_latent_min_height,
+ tile_latent_min_width,
+ tile_latent_stride_height,
+ tile_latent_stride_width,
+ ) = self._spatial_tile_geometry()
blend_height = tile_latent_min_height - tile_latent_stride_height
blend_width = tile_latent_min_width - tile_latent_stride_width
@@ -293,21 +294,15 @@ class ParallelTiledVAE(ABC, nn.Module, LayerwiseOffloadableModuleMixin):
world_size, rank = sp_group.world_size, sp_group.rank_in_group
_, _, T, H, W = z.shape
- tile_latent_min_height = (
- self.tile_sample_min_height // self.spatial_compression_ratio
- )
- tile_latent_min_width = (
- self.tile_sample_min_width // self.spatial_compression_ratio
- )
+ (
+ tile_latent_min_height,
+ tile_latent_min_width,
+ tile_latent_stride_height,
+ tile_latent_stride_width,
+ ) = self._spatial_tile_geometry()
tile_latent_min_num_frames = (
self.tile_sample_min_num_frames // self.temporal_compression_ratio
)
- tile_latent_stride_height = (
- self.tile_sample_stride_height // self.spatial_compression_ratio
- )
- tile_latent_stride_width = (
- self.tile_sample_stride_width // self.spatial_compression_ratio
- )
tile_latent_stride_num_frames = (
self.tile_sample_stride_num_frames // self.temporal_compression_ratio
)
@@ -436,18 +431,12 @@ class ParallelTiledVAE(ABC, nn.Module, LayerwiseOffloadableModuleMixin):
if world_size <= 1:
return self._decode(z)
- tile_latent_min_height = (
- self.tile_sample_min_height // self.spatial_compression_ratio
- )
- tile_latent_min_width = (
- self.tile_sample_min_width // self.spatial_compression_ratio
- )
- tile_latent_stride_height = (
- self.tile_sample_stride_height // self.spatial_compression_ratio
- )
- tile_latent_stride_width = (
- self.tile_sample_stride_width // self.spatial_compression_ratio
- )
+ (
+ tile_latent_min_height,
+ tile_latent_min_width,
+ tile_latent_stride_height,
+ tile_latent_stride_width,
+ ) = self._spatial_tile_geometry()
overlap_h = max(0, tile_latent_min_height - tile_latent_stride_height)
overlap_w = max(0, tile_latent_min_width - tile_latent_stride_width)
halo_h = overlap_h // 2
@@ -574,18 +563,12 @@ class ParallelTiledVAE(ABC, nn.Module, LayerwiseOffloadableModuleMixin):
# sample_height = height * self.spatial_compression_ratio
# sample_width = width * self.spatial_compression_ratio
- tile_latent_min_height = (
- self.tile_sample_min_height // self.spatial_compression_ratio
- )
- tile_latent_min_width = (
- self.tile_sample_min_width // self.spatial_compression_ratio
- )
- tile_latent_stride_height = (
- self.tile_sample_stride_height // self.spatial_compression_ratio
- )
- tile_latent_stride_width = (
- self.tile_sample_stride_width // self.spatial_compression_ratio
- )
+ (
+ tile_latent_min_height,
+ tile_latent_min_width,
+ tile_latent_stride_height,
+ tile_latent_stride_width,
+ ) = self._spatial_tile_geometry()
blend_height = self.tile_sample_min_height - self.tile_sample_stride_height
blend_width = self.tile_sample_min_width - self.tile_sample_stride_width
@@ -809,16 +792,5 @@ class DiagonalGaussianDistribution:
dim=dims,
)
- def nll(
- self, sample: torch.Tensor, dims: tuple[int, ...] = (1, 2, 3)
- ) -> torch.Tensor:
- if self.deterministic:
- return torch.Tensor([0.0])
- logtwopi = np.log(2.0 * np.pi)
- return 0.5 * torch.sum(
- logtwopi + self.logvar + torch.pow(sample - self.mean, 2) / self.var,
- dim=dims,
- )
-
def mode(self) -> torch.Tensor:
return self.mean
diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/cosmos3_avae.py b/python/sglang/multimodal_gen/runtime/models/vaes/cosmos3_avae.py
index ed6a5b054c..3d152a466e 100644
--- a/python/sglang/multimodal_gen/runtime/models/vaes/cosmos3_avae.py
+++ b/python/sglang/multimodal_gen/runtime/models/vaes/cosmos3_avae.py
@@ -193,12 +193,6 @@ class Cosmos3AVAEAudioTokenizer(nn.Module, LayerwiseOffloadableModuleMixin):
def temporal_compression_factor(self) -> int:
return self.hop_size
- def get_latent_num_samples(self, num_audio_samples: int) -> int:
- return int(num_audio_samples) // self.hop_size
-
- def get_audio_num_samples(self, num_latent_samples: int) -> int:
- return int(num_latent_samples) * self.hop_size
-
def _denormalize_latent(self, latent: torch.Tensor) -> torch.Tensor:
if self.normalization_type == "tanh":
in_dtype = latent.dtype
diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/dac.py b/python/sglang/multimodal_gen/runtime/models/vaes/dac.py
index 6c52484cc4..e3f1663c0e 100644
--- a/python/sglang/multimodal_gen/runtime/models/vaes/dac.py
+++ b/python/sglang/multimodal_gen/runtime/models/vaes/dac.py
@@ -370,7 +370,6 @@ class Encoder(nn.Module):
# Wrap black into nn.Sequential
self.block = nn.Sequential(*self.block)
- self.enc_dim = d_model
def forward(self, x):
return self.block(x)
diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/hunyuan3d_vae.py b/python/sglang/multimodal_gen/runtime/models/vaes/hunyuan3d_vae.py
index 8ad0a992a6..6bba427804 100644
--- a/python/sglang/multimodal_gen/runtime/models/vaes/hunyuan3d_vae.py
+++ b/python/sglang/multimodal_gen/runtime/models/vaes/hunyuan3d_vae.py
@@ -209,7 +209,6 @@ class QKVMultiheadCrossAttention(nn.Module):
):
super().__init__()
self.heads = heads
- self.n_data = n_data
self.q_norm = (
norm_layer(width // heads, elementwise_affine=True, eps=1e-6)
if qk_norm
@@ -255,7 +254,6 @@ class MultiheadCrossAttention(nn.Module):
kv_cache: bool = False,
):
super().__init__()
- self.n_data = n_data
self.width = width
self.heads = heads
self.data_width = width if data_width is None else data_width
@@ -338,7 +336,6 @@ class QKVMultiheadAttention(nn.Module):
):
super().__init__()
self.heads = heads
- self.n_ctx = n_ctx
self.q_norm = (
norm_layer(width // heads, elementwise_affine=True, eps=1e-6)
if qk_norm
@@ -381,7 +378,6 @@ class MultiheadAttention(nn.Module):
drop_path_rate: float = 0.0,
):
super().__init__()
- self.n_ctx = n_ctx
self.width = width
self.heads = heads
self.c_qkv = nn.Linear(width, width * 3, bias=qkv_bias)
@@ -450,7 +446,6 @@ class Transformer(nn.Module):
drop_path_rate: float = 0.0,
):
super().__init__()
- self.n_ctx = n_ctx
self.width = width
self.layers = layers
self.resblocks = nn.ModuleList(
@@ -512,7 +507,6 @@ class CrossAttentionDecoder(nn.Module):
if self.enable_ln_post:
self.ln_post = nn.LayerNorm(width)
self.output_proj = nn.Linear(width, out_channels)
- self.label_type = label_type
def set_cross_attention_processor(self, processor):
self.cross_attn_decoder.attn.attention.attn_processor = processor
diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/ltx_2_3_condition_encoder.py b/python/sglang/multimodal_gen/runtime/models/vaes/ltx_2_3_condition_encoder.py
index 4bd41d8aaa..1c23eed5d8 100644
--- a/python/sglang/multimodal_gen/runtime/models/vaes/ltx_2_3_condition_encoder.py
+++ b/python/sglang/multimodal_gen/runtime/models/vaes/ltx_2_3_condition_encoder.py
@@ -131,7 +131,6 @@ class LTX23VideoConditionEncoder(
latent_log_var = str(vae_config.get("latent_log_var", "uniform"))
self.patch_size = patch_size
- self.latency_channels = latent_channels
self.latent_log_var = latent_log_var
self.per_channel_statistics = LTX23PerChannelStatistics(latent_channels)
diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/ltx_2_audio.py b/python/sglang/multimodal_gen/runtime/models/vaes/ltx_2_audio.py
index 5d0bbfd997..7011ec5c55 100644
--- a/python/sglang/multimodal_gen/runtime/models/vaes/ltx_2_audio.py
+++ b/python/sglang/multimodal_gen/runtime/models/vaes/ltx_2_audio.py
@@ -349,7 +349,6 @@ class LTX2AudioAudioPatchifier:
):
self.hop_length = hop_length
self.sample_rate = sample_rate
- self.audio_latent_downsample_factor = audio_latent_downsample_factor
self.is_causal = is_causal
self._patch_size = (1, patch_size, patch_size)
@@ -413,7 +412,6 @@ class LTX2AudioEncoder(nn.Module):
base_block_channels = base_channels
base_resolution = resolution
- self.z_shape = (1, latent_channels, base_resolution, base_resolution)
if self.causality_axis is not None:
self.conv_in = LTX2AudioCausalConv2d(
@@ -594,7 +592,6 @@ class LTX2AudioDecoder(nn.Module):
base_block_channels = base_channels * self.channel_multipliers[-1]
base_resolution = resolution // (2 ** (self.num_resolutions - 1))
- self.z_shape = (1, latent_channels, base_resolution, base_resolution)
if self.causality_axis is not None:
self.conv_in = LTX2AudioCausalConv2d(
diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/ltx_2_vae.py b/python/sglang/multimodal_gen/runtime/models/vaes/ltx_2_vae.py
index 9905f4aa81..a45a50df82 100644
--- a/python/sglang/multimodal_gen/runtime/models/vaes/ltx_2_vae.py
+++ b/python/sglang/multimodal_gen/runtime/models/vaes/ltx_2_vae.py
@@ -122,9 +122,7 @@ class LTX2VideoCausalConv3d(nn.Module):
def _weight_is_channels_last_3d(self) -> bool:
w = self.conv.weight
- return hasattr(torch, "channels_last_3d") and _is_channels_last_3d_stride(
- tuple(w.size()), tuple(w.stride())
- )
+ return _is_channels_last_3d_stride(tuple(w.size()), tuple(w.stride()))
def _causal_temporal_pad_channels_last(
self,
@@ -1769,17 +1767,10 @@ class AutoencoderKLLTX2Video(ParallelTiledVAE):
# intermediate tiles together, the memory requirement can be lowered.
self.use_tiling = False
- # When decoding temporally long video latents, the memory requirement is very high. By decoding latent frames
- # at a fixed frame batch size (based on `self.num_latent_frames_batch_sizes`), the memory requirement can be lowered.
+ # Temporal tiling bounds memory when encoding or decoding long videos.
self.use_framewise_encoding = False
self.use_framewise_decoding = False
- # This can be configured based on the amount of GPU memory available.
- # `16` for sample frames and `2` for latent frames are sensible defaults for consumer GPUs.
- # Setting it to higher values results in higher memory usage.
- self.num_sample_frames_batch_size = 16
- self.num_latent_frames_batch_size = 2
-
# The minimal tile height and width for spatial tiling to be used
self.tile_sample_min_height = 512
self.tile_sample_min_width = 512
diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_audio_vae/audio_vae.py b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_audio_vae/audio_vae.py
index c1ec122087..77084a69fd 100644
--- a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_audio_vae/audio_vae.py
+++ b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_audio_vae/audio_vae.py
@@ -222,7 +222,6 @@ class Encoder(nn.Module):
# Wrap black into nn.Sequential
self.block = nn.Sequential(*self.block)
- self.enc_dim = d_model
def forward(self, x):
return self.block(x)
diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_audio_vae/bigvgan.py b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_audio_vae/bigvgan.py
index 14a720deb0..a8fbd8b582 100644
--- a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_audio_vae/bigvgan.py
+++ b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_audio_vae/bigvgan.py
@@ -49,7 +49,6 @@ class SnakeBeta(nn.Module):
self.alpha.requires_grad = alpha_trainable
self.beta.requires_grad = alpha_trainable
- self.no_div_by_zero = 0.000000001
def forward(self, x):
alpha = self.alpha.unsqueeze(0).unsqueeze(-1)
diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/klvae.py b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/klvae.py
index 3f11f2684a..eab7ee373a 100644
--- a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/klvae.py
+++ b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/klvae.py
@@ -1248,7 +1248,6 @@ class AutoencoderKLLegacy(AutoencoderKL):
self.use_3d_conv = use_3d_conv
self.causal_encoder = causal_encoder
self.causal_decoder = causal_decoder
- self.slidedec = self.causal_encoder and not self.causal_decoder
# some registered parameters for simplicity
self.vae_ratio = int(np.cumprod(space_down)[-1])
diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/vae_cnn.py b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/vae_cnn.py
index 1f72c67f70..9de936e653 100644
--- a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/vae_cnn.py
+++ b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/vae_cnn.py
@@ -182,8 +182,6 @@ class EncoderFCN3D(nn.Module):
else:
self.num_res_blocks = num_res_blocks
- self.space_down_factors = space_down
- self.time_down_factors = time_down
self.in_channels = in_channels
self.use_fused_norm = (
diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/wanvae.py b/python/sglang/multimodal_gen/runtime/models/vaes/wanvae.py
index 94adf90130..d8e602157b 100644
--- a/python/sglang/multimodal_gen/runtime/models/vaes/wanvae.py
+++ b/python/sglang/multimodal_gen/runtime/models/vaes/wanvae.py
@@ -79,7 +79,7 @@ first_chunk = contextvars.ContextVar("first_chunk", default=None)
def _channels_last_3d_supported_by_platform() -> bool:
- return hasattr(torch, "channels_last_3d") and (
+ return (
current_platform.is_cuda()
or current_platform.is_rocm()
or current_platform.is_xpu()
diff --git a/python/sglang/multimodal_gen/runtime/models/vlas/pi05_core.py b/python/sglang/multimodal_gen/runtime/models/vlas/pi05_core.py
index 791b3125fc..249473b949 100644
--- a/python/sglang/multimodal_gen/runtime/models/vlas/pi05_core.py
+++ b/python/sglang/multimodal_gen/runtime/models/vlas/pi05_core.py
@@ -117,7 +117,6 @@ class PiGemmaRMSNorm(nn.Module):
super().__init__()
self.eps = eps
self.dim = dim
- self.cond_dim = cond_dim
if cond_dim is None:
self.weight = nn.Parameter(torch.zeros(dim))
self.dense = None
@@ -1212,21 +1211,6 @@ class Pi05CoreModel(nn.Module):
persistent=False,
)
- def retain_runtime_components(
- self,
- role: Literal["all", "prefix", "action", "idle"],
- ) -> None:
- if role == "all":
- return
- if role in ("prefix", "idle"):
- self.paligemma_with_expert.gemma_expert = None
- self.action_in_proj = None
- self.action_out_proj = None
- self.time_mlp_in = None
- self.time_mlp_out = None
- if role in ("action", "idle"):
- self.paligemma_with_expert.paligemma = None
-
def prepare_attention_masks_4d(
self,
att_2d_masks: torch.Tensor,
diff --git a/python/sglang/multimodal_gen/runtime/models/vocoder/ltx_2_vocoder.py b/python/sglang/multimodal_gen/runtime/models/vocoder/ltx_2_vocoder.py
index 7e604ca5d0..ab343cb8ad 100644
--- a/python/sglang/multimodal_gen/runtime/models/vocoder/ltx_2_vocoder.py
+++ b/python/sglang/multimodal_gen/runtime/models/vocoder/ltx_2_vocoder.py
@@ -247,7 +247,6 @@ class ResBlock(nn.Module):
padding_mode: str = "same",
):
super().__init__()
- self.dilations = dilations
self.negative_slope = leaky_relu_negative_slope
self.convs1 = nn.ModuleList(
diff --git a/python/sglang/multimodal_gen/runtime/pipelines/diffusers_pipeline.py b/python/sglang/multimodal_gen/runtime/pipelines/diffusers_pipeline.py
index f2cac3ba7d..34a9a4181e 100644
--- a/python/sglang/multimodal_gen/runtime/pipelines/diffusers_pipeline.py
+++ b/python/sglang/multimodal_gen/runtime/pipelines/diffusers_pipeline.py
@@ -721,9 +721,6 @@ class DiffusersPipeline(ComposedPipelineBase):
stage=DiffusersExecutionStage(self.diffusers_pipe),
)
- def initialize_pipeline(self, server_args: ServerArgs) -> None:
- pass
-
def post_init(self) -> None:
"""Post initialization hook."""
if self.post_init_called:
diff --git a/python/sglang/multimodal_gen/runtime/pipelines/ltx_2_pipeline.py b/python/sglang/multimodal_gen/runtime/pipelines/ltx_2_pipeline.py
index 6a750db23d..aa1ee266bf 100644
--- a/python/sglang/multimodal_gen/runtime/pipelines/ltx_2_pipeline.py
+++ b/python/sglang/multimodal_gen/runtime/pipelines/ltx_2_pipeline.py
@@ -718,7 +718,6 @@ class LTX2TwoStagePipeline(_BaseLTX2Pipeline):
self._distilled_lora_path = distilled_lora_path
self._stage1_lora_path = server_args.lora_path
self._stage1_lora_scale = float(server_args.lora_scale)
- self._active_lora_phase = None
self._active_lora_signature = None
self._use_premerged_stage2_transformer = False
# set when original mode merges stage-1 distilled LoRA into the DiT base
@@ -808,7 +807,6 @@ class LTX2TwoStagePipeline(_BaseLTX2Pipeline):
self._stage1_distilled_in_base = True
self._stage1_distilled_base_strength = strength
- self._active_lora_phase = "stage1"
self._active_lora_signature = None
logger.info(
"Merged LTX-2 stage-1 distilled LoRA (strength=%.4f) into the DiT base; "
@@ -963,7 +961,6 @@ class LTX2TwoStagePipeline(_BaseLTX2Pipeline):
# A pre-distilled DiT has no LoRA to switch to and runs the same
# weights in both stages. Guarding here covers every caller.
if self._distilled_lora_path is None:
- self._active_lora_phase = phase
return
distilled_lora_strength = self._get_stage_distilled_lora_strength(phase, batch)
phase_signature = (phase, distilled_lora_strength)
@@ -972,7 +969,6 @@ class LTX2TwoStagePipeline(_BaseLTX2Pipeline):
if self._stage1_distilled_in_base:
if self._switch_lora_phase_base_merged(phase, distilled_lora_strength):
- self._active_lora_phase = phase
self._active_lora_signature = phase_signature
return
# Base was restored (stage-1 strength override); fall through to the
@@ -981,7 +977,6 @@ class LTX2TwoStagePipeline(_BaseLTX2Pipeline):
if self._ltx2_residency.enter_phase(
phase
) and self._can_short_circuit_lora_switch(phase, batch):
- self._active_lora_phase = phase
self._active_lora_signature = phase_signature
return
@@ -1015,7 +1010,6 @@ class LTX2TwoStagePipeline(_BaseLTX2Pipeline):
# two-stage pipeline immediately.
self.deactivate_lora_weights(target="transformer")
- self._active_lora_phase = phase
self._active_lora_signature = phase_signature
def create_pipeline_stages(self, server_args: ServerArgs):
diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/base.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/base.py
index 91d475b20a..3a9851a24d 100644
--- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/base.py
+++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/base.py
@@ -157,21 +157,12 @@ class PipelineStage(StageDedupMixin, ABC):
# Default implementation - no verification
return VerificationResult()
- def maybe_free_model_hooks(self):
- pass
-
def load_model(self):
"""
Load the model for the stage.
"""
pass
- def offload_model(self):
- """
- Offload the model for the stage.
- """
- pass
-
def iter_sequential_requests(
self, batch: Req, server_args: ServerArgs
) -> Iterator[Req]:
@@ -384,15 +375,6 @@ class PipelineStage(StageDedupMixin, ABC):
current_platform.device_type,
)
- def set_logging(self, enable: bool):
- """
- Enable or disable logging for this stage.
-
- Args:
- enable: Whether to enable logging.
- """
- self._enable_logging = enable
-
def __call__(
self,
batch: Req,
diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising.py
index e2f351037e..7de0fadc7d 100644
--- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising.py
+++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising.py
@@ -377,7 +377,6 @@ class DenoisingStage(PipelineStage, RolloutDenoisingMixin):
self.transformer_2 = transformer_2
# cache-dit state (for delayed mounting and idempotent control)
self._cache_dit_enabled = False
- self._cached_num_steps = None
# Per-request Cache-DiT overrides for the batch being executed
# (stashed by _maybe_enable_cache_dit; read by the config builders).
self._cache_dit_request_overrides: dict[str, Any] = {}
@@ -431,7 +430,6 @@ class DenoisingStage(PipelineStage, RolloutDenoisingMixin):
# misc
self.profiler = None
- self._is_warmed_up = False
self._extra_func_kwarg_names_cache: dict[int, tuple[bool, frozenset[str]]] = {}
def _infer_transformer_attention_backend(self) -> AttentionBackendEnum | None:
@@ -850,7 +848,6 @@ class DenoisingStage(PipelineStage, RolloutDenoisingMixin):
for transformer in filter(None, [self.transformer, self.transformer_2]):
disable_cache_on_transformer(transformer)
self._cache_dit_enabled = False
- self._cached_num_steps = None
self._cache_dit_active_key = None
def _cache_dit_secondary_uses_primary_config(self) -> bool:
@@ -1206,7 +1203,6 @@ class DenoisingStage(PipelineStage, RolloutDenoisingMixin):
)
self._cache_dit_enabled = True
- self._cached_num_steps = num_inference_steps
self._cache_dit_active_key = desired_key
@lru_cache(maxsize=8)
diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/latent_preparation.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/latent_preparation.py
index 6664bf01a5..88c1e2523f 100644
--- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/latent_preparation.py
+++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/latent_preparation.py
@@ -28,6 +28,29 @@ from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
logger = init_logger(__name__)
+def verify_latent_preparation_inputs(batch: Req) -> VerificationResult:
+ """Input contract shared by generic and model-specific latent preparation."""
+ result = VerificationResult()
+ result.add_check(
+ "prompt_or_embeds",
+ None,
+ lambda _: (
+ V.string_or_list_strings(batch.prompt)
+ or V.list_not_empty(batch.prompt_embeds)
+ ),
+ )
+ result.add_check("prompt_embeds", batch.prompt_embeds, V.list_of_tensors)
+ result.add_check(
+ "num_videos_per_prompt", batch.num_outputs_per_prompt, V.positive_int
+ )
+ result.add_check("generator", batch.generator, V.generator_or_list_generators)
+ result.add_check("num_frames", batch.num_frames, V.positive_int)
+ result.add_check("height", batch.height, V.positive_int)
+ result.add_check("width", batch.width, V.positive_int)
+ result.add_check("latents", batch.latents, V.none_or_tensor)
+ return result
+
+
@dataclass(frozen=True)
class LatentPreparationFingerprint:
height: int | None
@@ -352,25 +375,7 @@ class LatentPreparationStage(PipelineStage):
def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult:
"""Verify latent preparation stage inputs."""
- result = VerificationResult()
- result.add_check(
- "prompt_or_embeds",
- None,
- lambda _: (
- V.string_or_list_strings(batch.prompt)
- or V.list_not_empty(batch.prompt_embeds)
- ),
- )
- result.add_check("prompt_embeds", batch.prompt_embeds, V.list_of_tensors)
- result.add_check(
- "num_videos_per_prompt", batch.num_outputs_per_prompt, V.positive_int
- )
- result.add_check("generator", batch.generator, V.generator_or_list_generators)
- result.add_check("num_frames", batch.num_frames, V.positive_int)
- result.add_check("height", batch.height, V.positive_int)
- result.add_check("width", batch.width, V.positive_int)
- result.add_check("latents", batch.latents, V.none_or_tensor)
- return result
+ return verify_latent_preparation_inputs(batch)
def verify_output(self, batch: Req, server_args: ServerArgs) -> VerificationResult:
"""Verify latent preparation stage outputs."""
diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/glm_image.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/glm_image.py
index 5455eb4883..638ecad0ef 100644
--- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/glm_image.py
+++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/glm_image.py
@@ -906,14 +906,6 @@ class GlmImageBeforeDenoisingStage(PipelineStage):
)
self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor)
- self.default_sample_size = (
- self.transformer.config.sample_size
- if hasattr(self, "transformer")
- and self.transformer is not None
- and hasattr(self.transformer.config, "sample_size")
- else 128
- )
-
def component_uses(
self, server_args: ServerArgs, stage_name: str | None = None
) -> list[ComponentUse]:
diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/helios_denoising.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/helios_denoising.py
index aa10545429..91ef3fd9ef 100644
--- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/helios_denoising.py
+++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/helios_denoising.py
@@ -29,10 +29,7 @@ from sglang.multimodal_gen.runtime.pipelines_core.diffusion_scheduler_utils impo
get_or_create_request_scheduler,
)
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
-from sglang.multimodal_gen.runtime.pipelines_core.stages.base import (
- PipelineStage,
- StageParallelismType,
-)
+from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage
from sglang.multimodal_gen.runtime.server_args import ServerArgs
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
from sglang.multimodal_gen.runtime.utils.perf_logger import StageProfiler
@@ -139,10 +136,6 @@ class HeliosChunkedDenoisingStage(PipelineStage):
def role_affinity(self) -> RoleType:
return RoleType.DENOISER
- @property
- def parallelism_type(self):
- return StageParallelismType.REPLICATED
-
def component_uses(
self, server_args: ServerArgs, stage_name: str | None = None
) -> list[ComponentUse]:
diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/hunyuan3d/shape.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/hunyuan3d/shape.py
index 72cf0868c2..a5f7a74462 100644
--- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/hunyuan3d/shape.py
+++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/hunyuan3d/shape.py
@@ -252,9 +252,6 @@ class Hunyuan3DShapeBeforeDenoisingStage(PipelineStage):
class Hunyuan3DShapeDenoisingStage(DenoisingStage):
"""Denoising stage for Hunyuan3D shape generation."""
- def __init__(self, transformer: Any, scheduler: Any, **kwargs) -> None:
- super().__init__(transformer=transformer, scheduler=scheduler, **kwargs)
-
def _component_name_for_stage_module(self, module, default_name: str) -> str:
if module is self.transformer:
return "hy3dshape_model"
diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6/decoding.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6/decoding.py
index eddc22a675..096265c4a3 100644
--- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6/decoding.py
+++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6/decoding.py
@@ -25,9 +25,6 @@ from sglang.multimodal_gen.runtime.server_args import ServerArgs
class Kandinsky6DecodingStage(DecodingStage):
"""Convert video to channel-first for shared VAE decode; attach the decoded audio."""
- def __init__(self, vae, pipeline=None) -> None:
- super().__init__(vae=vae, pipeline=pipeline)
-
def forward(self, batch: Req, server_args: ServerArgs) -> OutputBatch:
if batch.latents is None:
raise ValueError("latents must be available before Kandinsky6 decoding.")
diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/tiling.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/tiling.py
index 2a620d0254..b9d9bbb6b4 100644
--- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/tiling.py
+++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/tiling.py
@@ -13,7 +13,6 @@ import torch
from torch.nn import functional
VAE_SPATIAL_FACTOR: int = 16
-VAE_TEMPORAL_FACTOR: int = 4
# trained base resolutions: visual_size -> [(H, W), ...]
RESOLUTIONS: dict[int, list[tuple[int, int]]] = {
diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/lingbot_video_moe/text_encoding.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/lingbot_video_moe/text_encoding.py
index 510269c718..128511c524 100644
--- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/lingbot_video_moe/text_encoding.py
+++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/lingbot_video_moe/text_encoding.py
@@ -34,8 +34,6 @@ PROMPT_TEMPLATE = (
"commentary or evaluations:<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n"
"<|im_start|>assistant\n"
)
-IMG_PROMPT_TEMPLATE = "<|vision_start|><|image_pad|><|vision_end|>"
-VIDEO_PROMPT_TEMPLATE = "<|vision_start|><|video_pad|><|vision_end|>"
class LingBotVideoTextEncodingStage(TextEncodingStage):
diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/request_validation.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/request_validation.py
index 707be26ffe..9a7db8e1de 100644
--- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/request_validation.py
+++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/request_validation.py
@@ -13,6 +13,7 @@ import math
from collections.abc import Mapping, Sequence
from typing import Any
+from sglang.multimodal_gen.configs.utils import optional_positive_finite_float
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.constants import (
MINIMAX_H3_MAX_DURATION_SECONDS,
MINIMAX_H3_MIN_DURATION_SECONDS,
@@ -55,17 +56,6 @@ def _require_int(value: Any, path: str) -> int:
return value
-def _optional_positive_finite_float(value: Any, path: str) -> float | None:
- if value is None:
- return None
- if isinstance(value, bool) or not isinstance(value, (int, float)):
- raise ValueError(f"{path} must be a number")
- normalized = float(value)
- if not math.isfinite(normalized) or normalized <= 0.0:
- raise ValueError(f"{path} must be a positive finite number")
- return normalized
-
-
def _optional_nonnegative_finite_float(value: Any, path: str) -> float | None:
if value is None:
return None
@@ -358,8 +348,8 @@ def minimax_h3_validate_canonical_request(
"conditions": normalized_conditions,
"target": normalized_target,
}
- normalized_flow_shift = _optional_positive_finite_float(flow_shift, "flow_shift")
- normalized_audio_flow_shift = _optional_positive_finite_float(
+ normalized_flow_shift = optional_positive_finite_float(flow_shift, "flow_shift")
+ normalized_audio_flow_shift = optional_positive_finite_float(
audio_flow_shift, "audio_flow_shift"
)
if normalized_flow_shift is not None:
diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/denoising.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/denoising.py
index f09ee81187..f44355ec2b 100644
--- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/denoising.py
+++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/denoising.py
@@ -560,7 +560,6 @@ class MiniMaxH3DenoisingStage(DenoisingStage):
# would send the next request down the refresh path with nothing
# mounted.
self._cache_dit_enabled = False
- self._cached_num_steps = None
self._cache_dit_active_key = None
self._minimax_h3_cache_mode = None
self._set_cache_dit_input_preservation(False)
diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/latent_preparation.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/latent_preparation.py
index a3e9f67f1e..593b6e2787 100644
--- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/latent_preparation.py
+++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/latent_preparation.py
@@ -5,6 +5,9 @@ import torch
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage
+from sglang.multimodal_gen.runtime.pipelines_core.stages.latent_preparation import (
+ verify_latent_preparation_inputs,
+)
from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import (
StageValidators as V,
)
@@ -141,25 +144,7 @@ class MiniMaxH3LatentPreparationStage(PipelineStage):
}
def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult:
- result = VerificationResult()
- result.add_check(
- "prompt_or_embeds",
- None,
- lambda _: (
- V.string_or_list_strings(batch.prompt)
- or V.list_not_empty(batch.prompt_embeds)
- ),
- )
- result.add_check("prompt_embeds", batch.prompt_embeds, V.list_of_tensors)
- result.add_check(
- "num_videos_per_prompt", batch.num_outputs_per_prompt, V.positive_int
- )
- result.add_check("generator", batch.generator, V.generator_or_list_generators)
- result.add_check("num_frames", batch.num_frames, V.positive_int)
- result.add_check("height", batch.height, V.positive_int)
- result.add_check("width", batch.width, V.positive_int)
- result.add_check("latents", batch.latents, V.none_or_tensor)
- return result
+ return verify_latent_preparation_inputs(batch)
def verify_output(self, batch: Req, server_args: ServerArgs) -> VerificationResult:
result = VerificationResult()
diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/mova.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/mova.py
index 5eace7fd2f..b1f9c39e06 100644
--- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/mova.py
+++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/mova.py
@@ -168,7 +168,6 @@ class MOVADenoisingStage(PipelineStage):
self.dual_tower_bridge = dual_tower_bridge
self.scheduler = scheduler
self._cache_dit_enabled = False
- self._cached_num_steps = None
self._torch_compiled = False
def component_uses(
diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/qwen_image_layered.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/qwen_image_layered.py
index 2f98078f7c..d983847241 100644
--- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/qwen_image_layered.py
+++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/qwen_image_layered.py
@@ -140,7 +140,6 @@ colors, textures, atmosphere, and so on\n - Identify the text clearly visible in
explanation, and highlight it in the caption with quotation marks\n3. Maintain authenticity and accuracy:\n - Avoid
generalizations\n - Describe all visible information in the image, while do not add information not explicitly shown in
the image\n<|vision_start|><|image_pad|><|vision_end|><|im_end|>\n<|im_start|>assistant\n"""
- self.default_sample_size = 128
def component_uses(
self, server_args: ServerArgs, stage_name: str | None = None
diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/progressive_resolution/wan.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/progressive_resolution/wan.py
index 5ca42e926f..00a21c8bb8 100644
--- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/progressive_resolution/wan.py
+++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/progressive_resolution/wan.py
@@ -67,21 +67,6 @@ class WanProgressiveDenoisingStage(ProgressiveDenoisingStage):
arch, "spatial_compression_ratio", 8
)
- # ------------------------------------------------------------------
- # Resolution-change hook (no-op for Wan T2V)
- # ------------------------------------------------------------------
-
- def _on_resolution_change(
- self,
- ctx,
- batch: Req,
- server_args: ServerArgs,
- new_h_pixel: int,
- new_w_pixel: int,
- ) -> None:
- """Wan T2V has no spatial positional embeddings that require updating."""
- pass
-
# ------------------------------------------------------------------
# Resolution alignment (Wan patch embedding requires even spatial dims)
# ------------------------------------------------------------------
diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/realtime/text_encoding.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/realtime/text_encoding.py
index 895fc2db23..7f7d6ddaa7 100644
--- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/realtime/text_encoding.py
+++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/realtime/text_encoding.py
@@ -38,22 +38,6 @@ def _normalize_prompt_value(
return value
-def _copy_tensor_list(
- value: list[torch.Tensor] | None,
-) -> list[torch.Tensor] | None:
- if value is None:
- return None
- return list(value)
-
-
-def _copy_seq_lens(
- value: list[list[int]] | None,
-) -> list[list[int]] | None:
- if value is None:
- return None
- return [list(seq_lens) for seq_lens in value]
-
-
_TEXT_CACHE_TENSOR_LIST_FIELDS = (
"prompt_embeds",
"pooled_embeds",
@@ -77,9 +61,11 @@ _TEXT_CACHE_DEFAULT_EMPTY_LIST_FIELDS = {
def _copy_text_cache_field(name: str, value):
+ if value is None:
+ return None
if name in _TEXT_CACHE_SEQ_LENS_FIELDS:
- return _copy_seq_lens(value)
- return _copy_tensor_list(value)
+ return [list(seq_lens) for seq_lens in value]
+ return list(value)
class RealtimeTextState(BaseRealtimeState):
diff --git a/python/sglang/multimodal_gen/runtime/platforms/cuda.py b/python/sglang/multimodal_gen/runtime/platforms/cuda.py
index f713786715..19cd076e75 100644
--- a/python/sglang/multimodal_gen/runtime/platforms/cuda.py
+++ b/python/sglang/multimodal_gen/runtime/platforms/cuda.py
@@ -590,15 +590,6 @@ class CudaPlatformBase(Platform):
def get_device_capability(cls, device_id: int = 0) -> DeviceCapability | None:
raise NotImplementedError
- @classmethod
- def get_device_name(cls, device_id: int = 0) -> str:
- raise NotImplementedError
-
- @classmethod
- @lru_cache(maxsize=1)
- def get_device_total_memory(cls, device_id: int = 0) -> int:
- raise NotImplementedError
-
@classmethod
def is_async_output_supported(cls, enforce_eager: bool | None) -> bool:
if enforce_eager:
diff --git a/python/sglang/multimodal_gen/runtime/platforms/interface.py b/python/sglang/multimodal_gen/runtime/platforms/interface.py
index 2de23d02f7..5026ffacb4 100644
--- a/python/sglang/multimodal_gen/runtime/platforms/interface.py
+++ b/python/sglang/multimodal_gen/runtime/platforms/interface.py
@@ -536,8 +536,3 @@ class Platform:
def tensor_on_device(self, t: torch.Tensor) -> bool:
"""Check if a tensor is on the current platform's device."""
return t.is_cuda
-
-
-class UnspecifiedPlatform(Platform):
- _enum = PlatformEnum.UNSPECIFIED
- device_type = ""
diff --git a/python/sglang/multimodal_gen/runtime/platforms/mps.py b/python/sglang/multimodal_gen/runtime/platforms/mps.py
index e4b25a4aaf..31f793cb67 100644
--- a/python/sglang/multimodal_gen/runtime/platforms/mps.py
+++ b/python/sglang/multimodal_gen/runtime/platforms/mps.py
@@ -10,7 +10,6 @@ from sglang.multimodal_gen.runtime.platforms import (
Platform,
PlatformEnum,
)
-from sglang.multimodal_gen.runtime.platforms.interface import DeviceCapability
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
# SPDX-License-Identifier: Apache-2.0
@@ -44,10 +43,6 @@ class MpsPlatform(Platform):
def set_device(cls, device: torch.device) -> None:
pass
- @classmethod
- def get_device_capability(cls, device_id: int = 0) -> DeviceCapability | None:
- return None
-
@classmethod
def get_device_name(cls, device_id: int = 0) -> str:
return "Apple Silicon MPS"
@@ -116,11 +111,6 @@ class MpsPlatform(Platform):
"sglang.multimodal_gen.runtime.layers.attention.backends.sdpa.SDPABackend"
)
- @classmethod
- def get_device_communicator_cls(cls) -> str:
- # Use base communicator for MPS
- return "sglang.multimodal_gen.runtime.distributed.device_communicators.base_device_communicator.DeviceCommunicatorBase"
-
@classmethod
def get_all_to_all_communicator_cls(cls) -> str:
return (
diff --git a/python/sglang/multimodal_gen/runtime/platforms/musa.py b/python/sglang/multimodal_gen/runtime/platforms/musa.py
index 8983594558..af8ad1ce53 100644
--- a/python/sglang/multimodal_gen/runtime/platforms/musa.py
+++ b/python/sglang/multimodal_gen/runtime/platforms/musa.py
@@ -84,15 +84,6 @@ class MusaPlatformBase(Platform):
def get_device_capability(cls, device_id: int = 0) -> DeviceCapability | None:
raise NotImplementedError
- @classmethod
- def get_device_name(cls, device_id: int = 0) -> str:
- raise NotImplementedError
-
- @classmethod
- @lru_cache(maxsize=1)
- def get_device_total_memory(cls, device_id: int = 0) -> int:
- raise NotImplementedError
-
@classmethod
def is_async_output_supported(cls, enforce_eager: bool | None) -> bool:
if enforce_eager:
diff --git a/python/sglang/multimodal_gen/runtime/platforms/npu.py b/python/sglang/multimodal_gen/runtime/platforms/npu.py
index 032cf71f43..11aaa2d082 100644
--- a/python/sglang/multimodal_gen/runtime/platforms/npu.py
+++ b/python/sglang/multimodal_gen/runtime/platforms/npu.py
@@ -10,7 +10,6 @@ import torch
from sglang.multimodal_gen import envs
from sglang.multimodal_gen.runtime.platforms.interface import (
AttentionBackendEnum,
- DeviceCapability,
Platform,
PlatformEnum,
)
@@ -53,10 +52,6 @@ class NPUPlatformBase(Platform):
def get_local_torch_device(cls) -> torch.device:
return torch.device(f"npu:{envs.LOCAL_RANK}")
- @classmethod
- def get_device_capability(cls, device_id: int = 0) -> DeviceCapability:
- return None
-
@classmethod
def get_device_name(cls, device_id: int = 0) -> str:
return str(torch.npu.get_device_name(device_id))
@@ -115,10 +110,6 @@ class NPUPlatformBase(Platform):
return free_gpu_memory / (1 << 30)
- @classmethod
- def log_warnings(cls) -> None:
- pass
-
@classmethod
def get_current_memory_usage(
cls, device: torch.types.Device | None = None
diff --git a/python/sglang/multimodal_gen/runtime/platforms/rocm.py b/python/sglang/multimodal_gen/runtime/platforms/rocm.py
index 654fc52ac9..d617b41ebd 100644
--- a/python/sglang/multimodal_gen/runtime/platforms/rocm.py
+++ b/python/sglang/multimodal_gen/runtime/platforms/rocm.py
@@ -78,10 +78,6 @@ class RocmPlatform(Platform):
return False
return True
- @classmethod
- def log_warnings(cls) -> None:
- pass # ROCm-specific warnings can be added here
-
@classmethod
def get_current_memory_usage(cls, device: torch.device | None = None) -> float:
torch.cuda.reset_peak_memory_stats(device)
diff --git a/python/sglang/multimodal_gen/runtime/platforms/xpu.py b/python/sglang/multimodal_gen/runtime/platforms/xpu.py
index 5fa2d52e5d..d893a1cf73 100644
--- a/python/sglang/multimodal_gen/runtime/platforms/xpu.py
+++ b/python/sglang/multimodal_gen/runtime/platforms/xpu.py
@@ -62,11 +62,6 @@ class XpuPlatform(Platform):
return False
return True
- @classmethod
- def log_warnings(cls) -> None:
- """Log any XPU-specific warnings."""
- pass
-
@classmethod
def get_current_memory_usage(
cls, device: torch.types.Device | None = None
@@ -186,12 +181,6 @@ class XpuPlatform(Platform):
"sglang.multimodal_gen.runtime.layers.attention.backends.sdpa.SDPABackend"
)
- @classmethod
- def get_device_communicator_cls(cls) -> str:
- """Get device communicator class for Intel XPU distributed communication."""
- # Use base communicator for now; can be updated to use oneCCL-based communicator
- return "sglang.multimodal_gen.runtime.distributed.device_communicators.base_device_communicator.DeviceCommunicatorBase"
-
@classmethod
def get_all_to_all_communicator_cls(cls) -> str:
return (
diff --git a/python/sglang/multimodal_gen/runtime/post_training/sp_utils.py b/python/sglang/multimodal_gen/runtime/post_training/sp_utils.py
index d05ad0924c..64c1c997e3 100644
--- a/python/sglang/multimodal_gen/runtime/post_training/sp_utils.py
+++ b/python/sglang/multimodal_gen/runtime/post_training/sp_utils.py
@@ -11,7 +11,6 @@ from sglang.multimodal_gen.runtime.distributed import (
)
from sglang.multimodal_gen.runtime.distributed.communication_op import (
sequence_model_parallel_all_gather,
- sequence_model_parallel_all_reduce,
)
@@ -36,14 +35,6 @@ def gather_stacked_latents_for_sp(
return gathered_flat_inputs.unflatten(0, (bsz, t_steps))
-def all_reduce_if_sp_sharded(batch, tensor: torch.Tensor) -> torch.Tensor:
- if not should_do_sp_collective(batch):
- return tensor
- tensor = tensor.to(get_local_torch_device())
- sequence_model_parallel_all_reduce(tensor)
- return tensor
-
-
def all_gather_if_sp_sharded(batch, x: torch.Tensor, dim: int = 0) -> torch.Tensor:
if not should_do_sp_collective(batch):
return x
diff --git a/python/sglang/multimodal_gen/runtime/postprocess/realesrgan_upscaler.py b/python/sglang/multimodal_gen/runtime/postprocess/realesrgan_upscaler.py
index 31ea6b5b48..1a8319062a 100644
--- a/python/sglang/multimodal_gen/runtime/postprocess/realesrgan_upscaler.py
+++ b/python/sglang/multimodal_gen/runtime/postprocess/realesrgan_upscaler.py
@@ -77,12 +77,7 @@ class SRVGGNetCompact(nn.Module):
act_type: str = "prelu",
):
super().__init__()
- self.num_in_ch = num_in_ch
- self.num_out_ch = num_out_ch
- self.num_feat = num_feat
- self.num_conv = num_conv
self.upscale = upscale
- self.act_type = act_type
self.body = nn.ModuleList()
# first conv
@@ -302,9 +297,6 @@ class UpscalerModel:
def dtype(self) -> torch.dtype:
return next(self.net.parameters()).dtype
- def _copy_input_to_device(self, frames: np.ndarray) -> torch.Tensor:
- return torch.from_numpy(frames).to(self.device)
-
def _preprocess_input_tensor(self, imgs_t: torch.Tensor) -> torch.Tensor:
imgs_t = imgs_t.permute(0, 3, 1, 2).to(dtype=self.dtype).mul_(1.0 / 255.0)
if self.device.type == "cuda":
@@ -316,10 +308,6 @@ class UpscalerModel:
out = out.permute(0, 2, 3, 1).clamp(0.0, 1.0).mul_(255.0)
return out.to(torch.uint8).contiguous()
- @staticmethod
- def _copy_output_to_host(out: torch.Tensor) -> np.ndarray:
- return out.cpu().numpy()
-
def _start_cuda_timer(self):
if self.device.type != "cuda":
return None
@@ -463,7 +451,7 @@ class UpscalerModel:
start_time = time.perf_counter()
h2d_timer = self._start_cuda_timer()
- imgs_t = self._copy_input_to_device(imgs)
+ imgs_t = torch.from_numpy(imgs).to(self.device)
self._stop_cuda_timer(h2d_timer)
h2d_wall_duration_s = time.perf_counter() - start_time
@@ -501,7 +489,7 @@ class UpscalerModel:
start_time = time.perf_counter()
output_d2h_timer = self._start_cuda_timer()
- out_np = self._copy_output_to_host(out)
+ out_np = out.cpu().numpy()
self._stop_cuda_timer(output_d2h_timer)
output_d2h_wall_duration_s = time.perf_counter() - start_time
diff --git a/python/sglang/multimodal_gen/runtime/postprocess/rife_interpolator.py b/python/sglang/multimodal_gen/runtime/postprocess/rife_interpolator.py
index 04c9831c00..f364b371a0 100644
--- a/python/sglang/multimodal_gen/runtime/postprocess/rife_interpolator.py
+++ b/python/sglang/multimodal_gen/runtime/postprocess/rife_interpolator.py
@@ -350,7 +350,6 @@ class FrameInterpolator:
def __init__(self, model_path: Optional[str] = None):
self._model_path = model_path
- self._resolved_path: Optional[str] = None
def _ensure_model_loaded(self) -> Model:
"""Load RIFE model weights.
@@ -368,8 +367,6 @@ class FrameInterpolator:
# Resolve: local path pass-through, HF repo ID → download & cache
model_path = maybe_download_model(model_path)
- self._resolved_path = model_path
-
if model_path in _MODEL_CACHE:
return _MODEL_CACHE[model_path]
diff --git a/python/sglang/multimodal_gen/runtime/realtime/video.py b/python/sglang/multimodal_gen/runtime/realtime/video.py
index 2c2786ee59..5782ed9e7a 100644
--- a/python/sglang/multimodal_gen/runtime/realtime/video.py
+++ b/python/sglang/multimodal_gen/runtime/realtime/video.py
@@ -26,7 +26,6 @@ RAW_RGBA_DELTA_GZIP_CONTENT_TYPE = "application/x-raw-rgba-delta-gzip"
WEBP_FRAME_CONTENT_TYPE = "image/webp"
JPEG_FRAME_CONTENT_TYPE = "image/jpeg"
RAW_RGB_CHANNELS = 3
-RAW_RGBA_CHANNELS = 4
_RAW_RGB_DELTA_GZIP_LEVEL = 0
diff --git a/python/sglang/multimodal_gen/runtime/server_args/disagg.py b/python/sglang/multimodal_gen/runtime/server_args/disagg.py
index 7453385a1d..0240ca9f57 100644
--- a/python/sglang/multimodal_gen/runtime/server_args/disagg.py
+++ b/python/sglang/multimodal_gen/runtime/server_args/disagg.py
@@ -3,7 +3,7 @@
from __future__ import annotations
-from typing import ClassVar, Literal
+from typing import ClassVar
from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
from sglang.multimodal_gen.runtime.utils.argparse import FlexibleArgumentParser
@@ -74,11 +74,6 @@ class DisaggServerArgsMixin:
host, self.scheduler_port + 1, "pool_control_advertised_endpoint"
)
- def resolved_role_device(self) -> Literal["cpu", "cuda"]:
- if self.disagg_role_device == "auto":
- return "cpu" if self.num_gpus <= 0 else "cuda"
- return self.disagg_role_device
-
@classmethod
def add_disagg_cli_args(cls, parser: FlexibleArgumentParser) -> None:
role_default = (
diff --git a/python/sglang/multimodal_gen/runtime/server_args/server_args.py b/python/sglang/multimodal_gen/runtime/server_args/server_args.py
index df8bc672bc..410bed3fa1 100644
--- a/python/sglang/multimodal_gen/runtime/server_args/server_args.py
+++ b/python/sglang/multimodal_gen/runtime/server_args/server_args.py
@@ -640,13 +640,6 @@ class ServerArgs(DisaggServerArgsMixin):
def broker_port(self) -> int:
return self.port + 1
- @property
- def is_local_mode(self) -> bool:
- """
- If no server is running when a generation task begins, 'local_mode' will be enabled: a dedicated server will be launched
- """
- return self.host is None or self.port is None
-
def _adjust_path(self):
expand_path_fields(self)
self._adjust_save_paths()
diff --git a/python/sglang/multimodal_gen/runtime/utils/common.py b/python/sglang/multimodal_gen/runtime/utils/common.py
index 8c285c32ab..3de6862fd0 100644
--- a/python/sglang/multimodal_gen/runtime/utils/common.py
+++ b/python/sglang/multimodal_gen/runtime/utils/common.py
@@ -3,7 +3,6 @@
import ipaddress
import logging
import os
-import platform
import socket
from functools import lru_cache
from typing import Any
@@ -268,19 +267,6 @@ def get_zmq_socket(
return socket, endpoint
-# https://pytorch.org/docs/stable/notes/hip.html#checking-for-hip
-
-
-@lru_cache(maxsize=1)
-def is_host_cpu_x86() -> bool:
- machine = platform.machine().lower()
- return (
- machine in ("x86_64", "amd64", "i386", "i686")
- and hasattr(torch, "cpu")
- and torch.cpu.is_available()
- )
-
-
# cuda
diff --git a/python/sglang/multimodal_gen/runtime/utils/hf_diffusers_utils.py b/python/sglang/multimodal_gen/runtime/utils/hf_diffusers_utils.py
index fef29346f0..3bbabdc533 100644
--- a/python/sglang/multimodal_gen/runtime/utils/hf_diffusers_utils.py
+++ b/python/sglang/multimodal_gen/runtime/utils/hf_diffusers_utils.py
@@ -18,7 +18,6 @@
# ==============================================================================
"""Utilities for Huggingface Transformers."""
-import contextlib
import glob
import json
import os
@@ -400,25 +399,6 @@ def _verify_diffusers_model_complete(path: str) -> bool:
)
-_CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = {
- # ChatGLMConfig.model_type: ChatGLMConfig,
- # DbrxConfig.model_type: DbrxConfig,
- # ExaoneConfig.model_type: ExaoneConfig,
- # Qwen2_5_VLConfig.model_type: Qwen2_5_VLConfig,
-}
-
-for name, cls in _CONFIG_REGISTRY.items():
- with contextlib.suppress(ValueError):
- AutoConfig.register(name, cls)
-
-
-def download_from_hf(model_path: str):
- if os.path.exists(model_path):
- return model_path
-
- return snapshot_download(model_path, allow_patterns=["*.json", "*.bin", "*.model"])
-
-
def get_hf_config(
component_model_path: str,
trust_remote_code: bool,
@@ -435,29 +415,12 @@ def get_hf_config(
revision=revision,
**kwargs,
)
- if config.model_type in _CONFIG_REGISTRY:
- config_class = _CONFIG_REGISTRY[config.model_type]
- config = config_class.from_pretrained(component_model_path, revision=revision)
- # NOTE(HandH1998): Qwen2VL requires `_name_or_path` attribute in `config`.
- config._name_or_path = component_model_path
if model_override_args:
config.update(model_override_args)
return config
-def get_config(
- model: str,
- trust_remote_code: bool,
- revision: Optional[str] = None,
- model_override_args: Optional[dict] = None,
- **kwargs,
-):
- return AutoConfig.from_pretrained(
- model, trust_remote_code=trust_remote_code, revision=revision, **kwargs
- )
-
-
def load_dict(file_path):
if not os.path.exists(file_path):
return {}
@@ -604,29 +567,6 @@ def get_diffusers_component_config(
return combined_config
-# Models don't use the same configuration key for determining the maximum
-# context length. Store them here so we can sanely check them.
-# NOTE: The ordering here is important. Some models have two of these and we
-# have a preference for which value gets used.
-CONTEXT_LENGTH_KEYS = [
- "max_sequence_length",
- "seq_length",
- "max_seq_len",
- "model_max_length",
- "max_position_embeddings",
-]
-
-
-def attach_additional_stop_token_ids(tokenizer):
- # Special handling for stop token <|eom_id|> generated by llama 3 tool use.
- if "<|eom_id|>" in tokenizer.get_added_vocab():
- tokenizer.additional_stop_token_ids = {
- tokenizer.get_added_vocab()["<|eom_id|>"]
- }
- else:
- tokenizer.additional_stop_token_ids = None
-
-
def maybe_download_lora(
model_name_or_path: str,
local_dir: str | None = None,
diff --git a/python/sglang/multimodal_gen/runtime/utils/logging_utils.py b/python/sglang/multimodal_gen/runtime/utils/logging_utils.py
index dceecd9fd9..c915ab0f3f 100644
--- a/python/sglang/multimodal_gen/runtime/utils/logging_utils.py
+++ b/python/sglang/multimodal_gen/runtime/utils/logging_utils.py
@@ -7,7 +7,6 @@
import argparse
import contextlib
import dataclasses
-import datetime
import inspect
import logging
import os
@@ -15,16 +14,11 @@ import sys
import time
from contextlib import contextmanager
from enum import Enum
-from functools import lru_cache, partial
+from functools import lru_cache
from logging import Logger
from types import MethodType
from typing import Any, cast
-import sglang.multimodal_gen.envs as envs
-
-SGLANG_DIFFUSION_LOGGING_LEVEL = envs.SGLANG_DIFFUSION_LOGGING_LEVEL
-SGLANG_DIFFUSION_LOGGING_PREFIX = envs.SGLANG_DIFFUSION_LOGGING_PREFIX
-
# color
CYAN = "\033[1;36m"
RED = "\033[91m"
@@ -32,45 +26,6 @@ GREEN = "\033[92m"
YELLOW = "\033[93m"
RESET = "\033[0;0m"
-_FORMAT = (
- f"{SGLANG_DIFFUSION_LOGGING_PREFIX}%(levelname)s %(asctime)s "
- "[%(filename)s: %(lineno)d] %(message)s"
-)
-
-# _FORMAT = "[%(asctime)s] %(message)s"
-_DATE_FORMAT = "%m-%d %H:%M:%S"
-
-DEFAULT_LOGGING_CONFIG = {
- "formatters": {
- "sgl_diffusion": {
- "class": "sglang.multimodal_gen.runtime.utils.logging_utils.ColoredFormatter",
- "datefmt": _DATE_FORMAT,
- "format": _FORMAT,
- },
- },
- "handlers": {
- "sgl_diffusion": {
- "class": "logging.StreamHandler",
- "formatter": "sgl_diffusion",
- "level": SGLANG_DIFFUSION_LOGGING_LEVEL,
- "stream": "ext://sys.stdout",
- },
- },
- "loggers": {
- "sgl_diffusion": {
- "handlers": ["sgl_diffusion"],
- "level": "WARNING",
- "propagate": False,
- },
- },
- "root": {
- "handlers": ["sgl_diffusion"],
- "level": "DEBUG",
- },
- "version": 1,
- "disable_existing_loggers": False,
-}
-
class ColoredFormatter(logging.Formatter):
"""A logging formatter that adds color to log levels."""
@@ -422,72 +377,6 @@ def _sanitize_for_logging(obj: Any, key_hint: str | None = None) -> Any:
return ""
-def _trace_calls(log_path, root_dir, frame, event, arg=None):
- if event in ["call", "return"]:
- # Extract the filename, line number, function name, and the code object
- filename = frame.f_code.co_filename
- lineno = frame.f_lineno
- func_name = frame.f_code.co_name
- if not filename.startswith(root_dir):
- # only log the functions in the sgl_diffusion root_dir
- return
- # Log every function call or return
- try:
- last_frame = frame.f_back
- if last_frame is not None:
- last_filename = last_frame.f_code.co_filename
- last_lineno = last_frame.f_lineno
- last_func_name = last_frame.f_code.co_name
- else:
- # initial frame
- last_filename = ""
- last_lineno = 0
- last_func_name = ""
- with open(log_path, "a") as f:
- ts = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")
- if event == "call":
- f.write(
- f"{ts} Call to"
- f" {func_name} in {filename}:{lineno}"
- f" from {last_func_name} in {last_filename}:"
- f"{last_lineno}\n"
- )
- else:
- f.write(
- f"{ts} Return from"
- f" {func_name} in {filename}:{lineno}"
- f" to {last_func_name} in {last_filename}:"
- f"{last_lineno}\n"
- )
- except NameError:
- # modules are deleted during shutdown
- pass
- return partial(_trace_calls, log_path, root_dir)
-
-
-def enable_trace_function_call(log_file_path: str, root_dir: str | None = None):
- """
- Enable tracing of every function call in code under `root_dir`.
- This is useful for debugging hangs or crashes.
- `log_file_path` is the path to the log file.
- `root_dir` is the root directory of the code to trace. If None, it is the
- sgl_diffusion root directory.
-
- Note that this call is thread-level, any threads calling this function
- will have the trace enabled. Other threads will not be affected.
- """
- logger.warning(
- "SGLANG_DIFFUSION_TRACE_FUNCTION is enabled. It will record every"
- " function executed by Python. This will slow down the code. It "
- "is suggested to be used for debugging hang or crashes only."
- )
- logger.info("Trace frame log is saved to %s", log_file_path)
- if root_dir is None:
- # by default, this is the sgl_diffusion root directory
- root_dir = os.path.dirname(os.path.dirname(__file__))
- sys.settrace(partial(_trace_calls, log_file_path, root_dir))
-
-
def set_uvicorn_logging_configs(server_args=None):
from uvicorn.config import LOGGING_CONFIG
@@ -587,17 +476,6 @@ def get_log_level() -> int:
return root.level
-def suppress_loggers(loggers_to_suppress: list[str], level: int = logging.WARNING):
- original_levels = {}
-
- for logger_name in loggers_to_suppress:
- logger = logging.getLogger(logger_name)
- original_levels[logger_name] = logger.level
- logger.setLevel(level)
-
- return original_levels
-
-
def globally_suppress_loggers():
# globally suppress some obsessive loggers
target_names = [
diff --git a/python/sglang/multimodal_gen/runtime/utils/mesh3d_utils.py b/python/sglang/multimodal_gen/runtime/utils/mesh3d_utils.py
index 2820002861..55919b2585 100644
--- a/python/sglang/multimodal_gen/runtime/utils/mesh3d_utils.py
+++ b/python/sglang/multimodal_gen/runtime/utils/mesh3d_utils.py
@@ -305,12 +305,10 @@ class MeshRender:
self.set_default_texture_resolution(texture_size)
self.camera_distance = camera_distance
- self.camera_type = camera_type
self.bake_angle_thres = 75
self.bake_unreliable_kernel_size = int(
(2 / 512) * max(self.default_resolution[0], self.default_resolution[1])
)
- self.bake_mode = bake_mode
# Set up camera projection matrix
if camera_type == "orth":
diff --git a/python/sglang/multimodal_gen/runtime/utils/quantization_utils.py b/python/sglang/multimodal_gen/runtime/utils/quantization_utils.py
index cd48f0666c..309fafd7a6 100644
--- a/python/sglang/multimodal_gen/runtime/utils/quantization_utils.py
+++ b/python/sglang/multimodal_gen/runtime/utils/quantization_utils.py
@@ -438,14 +438,6 @@ def normalize_flat_modelopt_quant_config(
return normalized
-def _infer_nvfp4_group_size_from_tensors(weight, scale) -> Optional[int]:
- """Infer NVFP4 group_size from serialized weight/scale tensor shapes."""
- return _infer_nvfp4_group_size_from_shapes(
- getattr(weight, "shape", ()),
- getattr(scale, "shape", ()),
- )
-
-
def _infer_nvfp4_group_size_from_shapes(weight_shape, scale_shape) -> Optional[int]:
weight_shape = tuple(weight_shape or ())
scale_shape = tuple(scale_shape or ())
@@ -741,7 +733,7 @@ def _canonicalize_modulation_exclude(module_name: str) -> str:
return module_name
-def _build_nvfp4_config_from_safetensors_files(
+def build_nvfp4_config_from_safetensors_list(
file_paths: list[str],
param_names_mapping_dict: Optional[dict] = None,
reverse_param_names_mapping_dict: Optional[dict] = None,
@@ -953,32 +945,3 @@ def _build_nvfp4_config_from_safetensors_files(
e,
)
return None
-
-
-def build_nvfp4_config_from_safetensors(
- file_path: str,
- param_names_mapping_dict: Optional[dict] = None,
- reverse_param_names_mapping_dict: Optional[dict] = None,
- fallback_group_size: Optional[int] = None,
-) -> Optional[QuantizationConfig]:
- """Backward-compatible wrapper for a single safetensors file."""
- return _build_nvfp4_config_from_safetensors_files(
- [file_path],
- param_names_mapping_dict,
- reverse_param_names_mapping_dict,
- fallback_group_size,
- )
-
-
-def build_nvfp4_config_from_safetensors_list(
- file_paths: list[str],
- param_names_mapping_dict: Optional[dict] = None,
- reverse_param_names_mapping_dict: Optional[dict] = None,
- fallback_group_size: Optional[int] = None,
-) -> Optional[QuantizationConfig]:
- return _build_nvfp4_config_from_safetensors_files(
- file_paths,
- param_names_mapping_dict,
- reverse_param_names_mapping_dict,
- fallback_group_size,
- )
diff --git a/python/sglang/multimodal_gen/runtime/utils/vision.py b/python/sglang/multimodal_gen/runtime/utils/vision.py
index d3610187ef..2c311cfa23 100644
--- a/python/sglang/multimodal_gen/runtime/utils/vision.py
+++ b/python/sglang/multimodal_gen/runtime/utils/vision.py
@@ -111,9 +111,7 @@ def load_image(
else:
# in-memory loading path
image = PIL.Image.open(BytesIO(srt_get_image_bytes(image)))
- elif isinstance(image, PIL.Image.Image):
- image = image
- else:
+ elif not isinstance(image, PIL.Image.Image):
raise ValueError(
"Incorrect format used for the image. Should be bytes, a URL, a local path, base64/data URL, or a PIL image."
)
diff --git a/python/sglang/multimodal_gen/test/single_test_file/test_disagg_server.py b/python/sglang/multimodal_gen/test/single_test_file/test_disagg_server.py
index 9e2d818820..8a1017241f 100755
--- a/python/sglang/multimodal_gen/test/single_test_file/test_disagg_server.py
+++ b/python/sglang/multimodal_gen/test/single_test_file/test_disagg_server.py
@@ -203,7 +203,7 @@ class DisaggCluster:
log = _LOG_DIR / f"disagg_{self.name}_server.log"
self._logs["server"] = log
# Role processes register their transfer work_endpoint with the
- # derived value ``tcp://0.0.0.0:`` (see disagg_args.py). The
+ # derived value ``tcp://0.0.0.0:`` (see server_args/disagg.py). The
# server head must advertise the same literal so ``_handle_register``'s
# endpoint_to_idx exact-string match succeeds.
cmd = [
diff --git a/python/sglang/multimodal_gen/test/test_utils.py b/python/sglang/multimodal_gen/test/test_utils.py
index 526bc97d55..df0e4ba14b 100644
--- a/python/sglang/multimodal_gen/test/test_utils.py
+++ b/python/sglang/multimodal_gen/test/test_utils.py
@@ -13,7 +13,7 @@ import time
import wave
from dataclasses import dataclass
from pathlib import Path
-from typing import TYPE_CHECKING, Any
+from typing import Any
from urllib.parse import urljoin
import cv2
@@ -30,9 +30,6 @@ from sglang.multimodal_gen.runtime.utils.perf_logger import (
get_diffusion_perf_log_dir,
)
-if TYPE_CHECKING:
- from sglang.multimodal_gen.test.server.testcase_configs import DiffusionTestCase
-
logger = init_logger(__name__)
# GT is read from @. A given SHA only exists in the repo it was
@@ -248,16 +245,6 @@ def is_image_url(image_path: str | Path | None) -> bool:
)
-def probe_port(host="127.0.0.1", port=30010, timeout=2.0) -> bool:
- with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
- s.settimeout(timeout)
- try:
- s.connect((host, port))
- return True
- except OSError:
- return False
-
-
def is_in_ci() -> bool:
return get_bool_env_var("SGLANG_IS_IN_CI")
@@ -311,16 +298,6 @@ def wait_for_server_health(
)
-def post_json(
- base_url: str,
- path: str,
- payload: dict,
- timeout: float = 300.0,
-) -> httpx.Response:
- """POST JSON to ```` and return the response."""
- return httpx.post(urljoin(base_url, path), json=payload, timeout=timeout)
-
-
def run_command(command: list[str]) -> bool:
"""Run a CLI command and return whether it succeeded."""
print(f"Running command: {' '.join(command)}", flush=True)
@@ -371,35 +348,6 @@ def query_gpu_mem_used_mib(gpu_index: int = 0, required: bool = False) -> int |
return None
-def require_gpu_mem_query(gpu_index: int = 0) -> int:
- """Same as :func:`query_gpu_mem_used_mib` but asserts availability.
-
- Raises ``AssertionError`` when ``nvidia-smi`` is unavailable instead of
- returning ``None``, so callers can rely on a valid ``int`` result.
- """
- mem = query_gpu_mem_used_mib(gpu_index, required=True)
- assert mem is not None
- return mem
-
-
-def assert_gpu_mem_changed(
- label: str,
- before_mib: int,
- after_mib: int,
- min_delta_mib: int,
-) -> None:
- """Assert that GPU memory changed by at least *min_delta_mib* MiB."""
- delta = abs(after_mib - before_mib)
- logger.debug(
- f"[MEM] {label}: before={before_mib} MiB after={after_mib} MiB |delta|={delta} MiB"
- )
- assert delta >= min_delta_mib, (
- f"GPU memory change too small for '{label}': "
- f"|after-before|={delta} MiB < {min_delta_mib} MiB "
- f"(before={before_mib} MiB, after={after_mib} MiB)"
- )
-
-
def is_mp4(data: bytes) -> bool:
"""Check if data represents a valid MP4 file by magic bytes."""
if len(data) < 8:
@@ -454,16 +402,6 @@ def get_expected_image_format(
return "jpg" # Default
-def wait_for_port(host="127.0.0.1", port=30010, deadline=300.0, interval=0.5):
- end = time.time() + deadline
- last_err = None
- while time.time() < end:
- if probe_port(host, port, timeout=interval):
- return True
- time.sleep(interval)
- raise TimeoutError(f"Port {host}:{port} not ready. Last error: {last_err}")
-
-
def check_image_size(ut, image, width, height):
# check image size
ut.assertEqual(image.size, (width, height))
@@ -549,13 +487,6 @@ def validate_image(b64_json: str) -> None:
assert is_png(image_bytes) or is_jpeg(image_bytes), "Image must be PNG or JPEG"
-def validate_video(b64_json: str) -> None:
- """Decode and validate that video is a valid format."""
- video_bytes = base64.b64decode(b64_json)
- is_webm = video_bytes[:4] == b"\x1a\x45\xdf\xa3"
- assert is_mp4(video_bytes) or is_webm, "Video must be MP4 or WebM"
-
-
def validate_openai_video(video_bytes: bytes) -> None:
"""Validate that video is MP4 or WebM by magic bytes."""
is_webm = video_bytes.startswith(b"\x1a\x45\xdf\xa3")
@@ -1159,18 +1090,6 @@ def compare_audio_with_gt(
)
-def get_clip_threshold(
- case: "DiffusionTestCase",
- metadata: dict[str, Any] | None = None,
-) -> float:
- """Get CLIP similarity threshold for a consistency test case."""
- return get_consistency_thresholds(
- case_id=case.id,
- is_video=case.server_args.modality == "video",
- metadata=metadata,
- ).clip_threshold
-
-
@dataclass
class FrameConsistencyMetrics:
frame_index: int
@@ -1981,21 +1900,6 @@ def load_audio_consistency_gt(case_id: str, num_gpus: int) -> np.ndarray:
return loaded_gt
-def load_gt_embeddings(
- case_id: str,
- num_gpus: int,
- is_video: bool = False,
- output_format: str | None = None,
-) -> list[np.ndarray]:
- """Load GT images and convert them into CLIP embeddings."""
- return load_consistency_gt(
- case_id=case_id,
- num_gpus=num_gpus,
- is_video=is_video,
- output_format=output_format,
- ).embeddings
-
-
def gt_exists(
case_id: str,
num_gpus: int,
diff --git a/python/sglang/multimodal_gen/test/unit/test_cache_dit_per_request.py b/python/sglang/multimodal_gen/test/unit/test_cache_dit_per_request.py
index bde9d6513a..a3a08c4522 100644
--- a/python/sglang/multimodal_gen/test/unit/test_cache_dit_per_request.py
+++ b/python/sglang/multimodal_gen/test/unit/test_cache_dit_per_request.py
@@ -122,7 +122,6 @@ class TestPerRequestCacheDitTransitions(unittest.TestCase):
self.stage.transformer = object()
self.stage.transformer_2 = None
self.stage._cache_dit_enabled = False
- self.stage._cached_num_steps = None
self.stage._cache_dit_request_overrides = {}
self.stage._cache_dit_active_key = None
diff --git a/python/sglang/multimodal_gen/test/unit/test_server_args.py b/python/sglang/multimodal_gen/test/unit/test_server_args.py
index e64ee3d2c2..6688d324ca 100644
--- a/python/sglang/multimodal_gen/test/unit/test_server_args.py
+++ b/python/sglang/multimodal_gen/test/unit/test_server_args.py
@@ -3322,18 +3322,6 @@ class TestPerRoleParallelism(unittest.TestCase):
self.assertEqual(args.get_role_parallelism(RoleType.DENOISER)["tp_size"], 2)
self.assertEqual(args.get_role_parallelism(RoleType.DECODER)["sp_degree"], 4)
- def test_disagg_args_import_path_matches_server_args_package(self):
- from sglang.multimodal_gen.runtime.disaggregation import disagg_args
- from sglang.multimodal_gen.runtime.server_args.disagg import (
- DisaggServerArgsMixin,
- )
-
- self.assertIs(disagg_args.DisaggArgsMixin, DisaggServerArgsMixin)
- self.assertIs(
- disagg_args.DISAGG_RESULT_PORT_OFFSETS,
- DisaggServerArgsMixin.DISAGG_RESULT_PORT_OFFSETS,
- )
-
def test_gpu_ids_normalize_lists_and_commas(self):
args = self._from_dict({"model_path": "/fake", "gpu_ids": ["0,1", "6", "7 8"]})