[diffusion] refactor: remove unused helpers and write-only state (#43064)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
Mick
2026-10-09 16:30:55 +08:00
committed by GitHub
co-authored by Mick Qian
parent ee4e2f7818
commit 17b2f35ae7
150 changed files with 597 additions and 2456 deletions
@@ -3,14 +3,11 @@ SGLang Diffusion Server API client.
Provides a low-level interface for interacting with SGLang Diffusion HTTP server.
"""
import base64
import io
import os
import time
from typing import Any, Dict, Optional
import requests
from PIL import Image
class SGLDiffusionServerAPI:
@@ -417,38 +414,6 @@ class SGLDiffusionServerAPI:
}
return content_types.get(ext, "image/png")
def decode_image_from_response(
self, response_data: Dict[str, Any], index: int = 0
) -> Image.Image:
"""
Decode base64 image from API response.
Args:
response_data: API response dictionary
index: Index of the image in the response (default: 0)
Returns:
PIL Image object
"""
if "data" not in response_data or not response_data["data"]:
raise ValueError("No image data in response")
if index >= len(response_data["data"]):
raise IndexError(f"Image index {index} out of range")
image_data = response_data["data"][index]
if "b64_json" not in image_data or not image_data["b64_json"]:
raise ValueError("No base64 image data found")
image_bytes = base64.b64decode(image_data["b64_json"])
image = Image.open(io.BytesIO(image_bytes))
# Convert to RGB if needed
if image.mode != "RGB":
image = image.convert("RGB")
return image
def set_lora(
self,
lora_nickname: str,
@@ -1,4 +1,3 @@
import argparse
import os
from sglang.multimodal_gen.apps.webui.minimax_h3 import (
@@ -21,13 +20,6 @@ from sglang.srt.environ import envs
logger = init_logger(__name__)
def add_webui_args(parser: argparse.ArgumentParser):
"""Add the arguments for the generate command."""
parser = ServerArgs.add_cli_args(parser)
parser = SamplingParams.add_cli_args(parser)
return parser
def run_sgl_diffusion_webui(server_args: ServerArgs):
if is_minimax_h3(server_args):
return run_minimax_h3_webui(server_args)
@@ -65,11 +65,6 @@ class Krea2ArchConfig(DiTArchConfig):
def head_dim(self) -> int:
return self.features // self.heads
@property
def in_features_packed(self) -> int:
"""Patch-embed input width: channels * patch**2."""
return self.channels * self.patch**2
@dataclass
class Krea2DitConfig(DiTConfig):
@@ -17,12 +17,6 @@ class LTXModelType(Enum):
VideoOnly = "ltx video only model"
AudioOnly = "ltx audio only model"
def is_video_enabled(self) -> bool:
return self in (LTXModelType.AudioVideo, LTXModelType.VideoOnly)
def is_audio_enabled(self) -> bool:
return self in (LTXModelType.AudioVideo, LTXModelType.AudioOnly)
class LTX2RopeType(str, Enum):
"""
@@ -31,7 +31,4 @@ class Ideogram4TextEncoderConfig(Qwen3VLConfig):
self.arch_config.ideogram_fp8_weight_only = not use_bitsandbytes
self.arch_config.requires_gpu_resident_text_encoder = use_bitsandbytes
def finalize_model_arch(self):
self.post_diffusers_config_update()
arch_config: Qwen3VLArchConfig = field(default_factory=Qwen3VLArchConfig)
@@ -16,6 +16,19 @@ AUTO_PARALLEL_DECODE_MODE = "auto"
SPATIAL_SHARD_PARALLEL_DECODE_MODES = ("spatial_shard", "spatial")
def get_channelwise_decode_scale_and_shift(arch_config, device, dtype):
"""Return reciprocal std and mean broadcast over BCTHW latents."""
scaling_factor = 1.0 / torch.tensor(arch_config.latents_std, device=device).view(
1, arch_config.z_dim, 1, 1, 1
).to(device, dtype)
shift_factor = (
torch.tensor(arch_config.latents_mean)
.view(1, arch_config.z_dim, 1, 1, 1)
.to(device, dtype)
)
return scaling_factor, shift_factor
@lru_cache(maxsize=8)
def is_spatial_shard_parallel_decode_mode(mode: str) -> bool:
return mode in SPATIAL_SHARD_PARALLEL_DECODE_MODES
@@ -2,9 +2,38 @@
from dataclasses import dataclass, field
from typing import Any, List
import torch
from sglang.multimodal_gen.configs.models.vaes.base import VAEArchConfig, VAEConfig
def get_ltx_video_decode_scale_and_shift(device, dtype, vae, arch_config):
"""Invert LTX video normalization: (z - mean) * scaling_factor / std."""
latents_mean = getattr(vae, "latents_mean", None)
latents_std = getattr(vae, "latents_std", None)
scaling_factor = (
getattr(getattr(vae, "config", None), "scaling_factor", None)
or getattr(vae, "scaling_factor", None)
or getattr(arch_config, "scaling_factor", None)
or 1.0
)
if isinstance(scaling_factor, (int, float)) and float(scaling_factor) == 0.0:
scaling_factor = 1.0
if isinstance(latents_mean, torch.Tensor) and isinstance(latents_std, torch.Tensor):
latents_mean = latents_mean.to(device=device, dtype=dtype).view(1, -1, 1, 1, 1)
latents_std = latents_std.to(device=device, dtype=dtype).view(1, -1, 1, 1, 1)
sf = torch.tensor(float(scaling_factor), device=device, dtype=dtype).view(
1, 1, 1, 1, 1
)
return sf / latents_std, latents_mean
sf = torch.tensor(float(scaling_factor), device=device, dtype=dtype).view(
1, 1, 1, 1, 1
)
return sf, None
@dataclass
class LTXVideoVAEArchConfig(VAEArchConfig):
# Architecture params
@@ -5,9 +5,8 @@ import json
import math
import os
from collections.abc import Callable
from dataclasses import asdict, dataclass, field, fields
from dataclasses import dataclass, field, fields
from enum import Enum
from operator import attrgetter
from typing import Any, ClassVar
import numpy as np
@@ -56,6 +55,16 @@ class STA_Mode(str, Enum):
NONE = None
def pack_latents_2x2(latents, batch_size, num_channels_latents, height, width):
latents = latents.view(
batch_size, num_channels_latents, height // 2, 2, width // 2, 2
)
latents = latents.permute(0, 2, 4, 1, 3, 5)
return latents.reshape(
batch_size, (height // 2) * (width // 2), num_channels_latents * 4
)
def postprocess_text(output: BaseEncoderOutput, _text_inputs) -> torch.tensor:
raise NotImplementedError
@@ -1198,35 +1207,6 @@ class PipelineConfig:
f"Length of text postprocess functions ({len(self.postprocess_text_funcs)}) must be equal to length of text preprocessing functions ({len(self.preprocess_text_funcs)})"
)
def dump_to_json(self, file_path: str):
output_dict = {f.name: attrgetter(f.name)(self) for f in fields(self)}
del_keys = []
for key, value in output_dict.items():
if isinstance(value, ModelConfig):
model_dict = asdict(value)
# Model Arch Config should be hidden away from the users
model_dict.pop("arch_config")
output_dict[key] = model_dict
elif isinstance(value, tuple) and all(
isinstance(v, ModelConfig) for v in value
):
model_dicts = []
for v in value:
model_dict = asdict(v)
# Model Arch Config should be hidden away from the users
model_dict.pop("arch_config")
model_dicts.append(model_dict)
output_dict[key] = model_dicts
elif isinstance(value, tuple) and all(callable(f) for f in value):
# Skip dumping functions
del_keys.append(key)
for key in del_keys:
output_dict.pop(key, None)
with open(file_path, "w") as f:
json.dump(output_dict, f, indent=2)
def load_from_json(self, file_path: str):
with open(file_path) as f:
input_pipeline_dict = json.load(f)
@@ -67,12 +67,6 @@ class DiffusersGenericPipelineConfig(PipelineConfig):
"""
pass
def adjust_size(self, width, height, image):
"""
Pass through - diffusers handles size adjustments.
"""
return width, height
# Static subclasses for each non-default task type.
# These exist so that _get_diffusers_model_info() can swap the task_type without
@@ -20,12 +20,12 @@ from sglang.multimodal_gen.configs.models.vaes.flux import Flux2VAEConfig, FluxV
from sglang.multimodal_gen.configs.pipeline_configs.base import (
ImagePipelineConfig,
ModelTaskType,
pack_latents_2x2,
shard_rotary_emb_for_sp,
)
from sglang.multimodal_gen.configs.pipeline_configs.hunyuan import (
clip_postprocess_text,
)
from sglang.multimodal_gen.configs.pipeline_configs.qwen_image import _pack_latents
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
@@ -141,7 +141,9 @@ class FluxPipelineConfig(ImagePipelineConfig):
width = 2 * (batch.width // (self.vae_config.arch_config.vae_scale_factor * 2))
num_channels_latents = self.dit_config.arch_config.in_channels // 4
# pack latents
return _pack_latents(latents, batch_size, num_channels_latents, height, width)
return pack_latents_2x2(
latents, batch_size, num_channels_latents, height, width
)
def get_pos_prompt_embeds(self, batch):
return batch.prompt_embeds[1]
@@ -11,6 +11,9 @@ from sglang.multimodal_gen.configs.models import DiTConfig, EncoderConfig, VAECo
from sglang.multimodal_gen.configs.models.dits.joy_image import JoyImageDiTConfig
from sglang.multimodal_gen.configs.models.encoders.qwen3vl import Qwen3VLConfig
from sglang.multimodal_gen.configs.models.vaes import WanVAEConfig
from sglang.multimodal_gen.configs.models.vaes.base import (
get_channelwise_decode_scale_and_shift,
)
from sglang.multimodal_gen.configs.pipeline_configs.base import (
ImagePipelineConfig,
ModelTaskType,
@@ -249,32 +252,10 @@ class JoyImageEditPipelineConfig(ImagePipelineConfig):
def get_decode_scale_and_shift(
self, device, dtype, vae
) -> Tuple[torch.Tensor, torch.Tensor]:
"""Get VAE denormalization scale and shift.
Args:
device: Target device
dtype: Target dtype
vae: VAE model
Returns:
Tuple of (scaling_factor, shift_factor)
"""
vae_arch_config = self.vae_config.arch_config
# Create scale factor: 1.0 / std
scaling_factor = 1.0 / torch.tensor(
vae_arch_config.latents_std, device=device
).view(1, vae_arch_config.z_dim, 1, 1, 1).to(device, dtype)
# Create shift factor: mean
shift_factor = (
torch.tensor(vae_arch_config.latents_mean)
.view(1, vae_arch_config.z_dim, 1, 1, 1)
.to(device, dtype)
return get_channelwise_decode_scale_and_shift(
self.vae_config.arch_config, device, dtype
)
return scaling_factor, shift_factor
def prepare_calculated_size(self, img: Image.Image) -> Tuple[int, int]:
img_h, img_w = img.size[1], img.size[0] # PIL (w,h)
bucket = self.find_best_bucket((1, 1, img_h, img_w))
@@ -6,6 +6,9 @@ from diffusers.image_processor import VaeImageProcessor
from sglang.multimodal_gen.configs.models import DiTConfig, VAEConfig
from sglang.multimodal_gen.configs.models.dits.krea2 import Krea2DitConfig
from sglang.multimodal_gen.configs.models.vaes.base import (
get_channelwise_decode_scale_and_shift,
)
from sglang.multimodal_gen.configs.models.vaes.qwenimage import QwenImageVAEConfig
from sglang.multimodal_gen.configs.pipeline_configs.base import (
ImagePipelineConfig,
@@ -146,16 +149,9 @@ class Krea2PipelineConfig(ImagePipelineConfig):
# --- VAE decode (same as Qwen-Image: latents * std + mean) ---
def get_decode_scale_and_shift(self, device, dtype, vae):
vae_arch = self.vae_config.arch_config
scaling_factor = 1.0 / torch.tensor(vae_arch.latents_std, device=device).view(
1, vae_arch.z_dim, 1, 1, 1
).to(device, dtype)
shift_factor = (
torch.tensor(vae_arch.latents_mean)
.view(1, vae_arch.z_dim, 1, 1, 1)
.to(device, dtype)
return get_channelwise_decode_scale_and_shift(
self.vae_config.arch_config, device, dtype
)
return scaling_factor, shift_factor
def post_denoising_loop(self, latents, batch):
latents, batch_size, channels, height, width = self._unpad_and_unpack_latents(
@@ -17,6 +17,7 @@ from sglang.multimodal_gen.configs.pipeline_configs.base import (
ImagePipelineConfig,
ModelTaskType,
TextConditioningOutput,
pack_latents_2x2,
)
from sglang.multimodal_gen.configs.pipeline_configs.model_deployment_config import (
ModelDeploymentConfig,
@@ -171,17 +172,6 @@ def _unpack_latents(latents, height, width, vae_scale_factor):
return latents
def _pack_latents(latents, batch_size, num_channels_latents, height, width):
latents = latents.view(
batch_size, num_channels_latents, height // 2, 2, width // 2, 2
)
latents = latents.permute(0, 2, 4, 1, 3, 5)
latents = latents.reshape(
batch_size, (height // 2) * (width // 2), num_channels_latents * 4
)
return latents
def _calculate_shift(
image_seq_len,
base_seq_len: int = 256,
@@ -293,7 +283,7 @@ class LongCatImagePipelineConfig(ImagePipelineConfig):
def maybe_pack_latents(self, latents, batch_size, batch):
num_channels_latents = self.dit_config.arch_config.num_channels_latents
_, _, h, w = latents.shape
return _pack_latents(latents, batch_size, num_channels_latents, h, w)
return pack_latents_2x2(latents, batch_size, num_channels_latents, h, w)
def maybe_prepare_latent_ids(self, latents):
# latents shape after packing: [B, (h//2)*(w//2), C*4]
@@ -517,7 +507,7 @@ class LongCatImageEditPipelineConfig(LongCatImagePipelineConfig):
batch_size // latent_condition.shape[0], 1, 1, 1
)
_, num_channels_latents, height, width = latent_condition.shape
return _pack_latents(
return pack_latents_2x2(
latent_condition, batch_size, num_channels_latents, height, width
)
@@ -11,7 +11,10 @@ from sglang.multimodal_gen.configs.models.encoders import (
)
from sglang.multimodal_gen.configs.models.encoders.gemma_3 import Gemma3Config
from sglang.multimodal_gen.configs.models.vaes.ltx_audio import LTXAudioVAEConfig
from sglang.multimodal_gen.configs.models.vaes.ltx_video import LTXVideoVAEConfig
from sglang.multimodal_gen.configs.models.vaes.ltx_video import (
LTXVideoVAEConfig,
get_ltx_video_decode_scale_and_shift,
)
from sglang.multimodal_gen.configs.pipeline_configs.base import (
ModelTaskType,
PipelineConfig,
@@ -564,36 +567,9 @@ class LTX2PipelineConfig(PipelineConfig):
)
def get_decode_scale_and_shift(self, device, dtype, vae):
latents_mean = getattr(vae, "latents_mean", None)
latents_std = getattr(vae, "latents_std", None)
scaling_factor = (
getattr(getattr(vae, "config", None), "scaling_factor", None)
or getattr(vae, "scaling_factor", None)
or getattr(self.vae_config.arch_config, "scaling_factor", None)
or 1.0
return get_ltx_video_decode_scale_and_shift(
device, dtype, vae, self.vae_config.arch_config
)
if isinstance(scaling_factor, (int, float)) and float(scaling_factor) == 0.0:
scaling_factor = 1.0
if isinstance(latents_mean, torch.Tensor) and isinstance(
latents_std, torch.Tensor
):
latents_mean = latents_mean.to(device=device, dtype=dtype).view(
1, -1, 1, 1, 1
)
latents_std = latents_std.to(device=device, dtype=dtype).view(
1, -1, 1, 1, 1
)
sf = torch.tensor(float(scaling_factor), device=device, dtype=dtype).view(
1, 1, 1, 1, 1
)
return sf / latents_std, latents_mean
sf = torch.tensor(float(scaling_factor), device=device, dtype=dtype).view(
1, 1, 1, 1, 1
)
return sf, None
@staticmethod
def _unpack_latents(
@@ -626,29 +602,6 @@ class LTX2PipelineConfig(PipelineConfig):
)
return latents
@staticmethod
def _denormalize_latents(
latents: torch.Tensor,
latents_mean: torch.Tensor,
latents_std: torch.Tensor,
scaling_factor: float = 1.0,
) -> torch.Tensor:
# Denormalize latents across the channel dimension [B, C, F, H, W]
latents_mean = latents_mean.view(1, -1, 1, 1, 1).to(
latents.device, latents.dtype
)
latents_std = latents_std.view(1, -1, 1, 1, 1).to(latents.device, latents.dtype)
latents = latents * latents_std / scaling_factor + latents_mean
return latents
@staticmethod
def _denormalize_audio_latents(
latents: torch.Tensor, latents_mean: torch.Tensor, latents_std: torch.Tensor
):
latents_mean = latents_mean.to(latents.device, latents.dtype)
latents_std = latents_std.to(latents.device, latents.dtype)
return (latents * latents_std) + latents_mean
@staticmethod
def _unpack_audio_latents(
latents: torch.Tensor,
@@ -70,16 +70,9 @@ class MiniMaxH3PipelineConfig(PipelineConfig):
output_audio_channels: int | None = 2
output_av_drift_tolerance_s: float | None = 0.25
def accepts_audio_input(self) -> bool:
return True
def supports_disaggregation(self) -> bool:
return False
@property
def requires_audio_output(self) -> bool:
return True
def get_model_deployment_config(self) -> ModelDeploymentConfig:
return ModelDeploymentConfig(
speed_mode_enable_torch_compile_by_default=False,
@@ -154,19 +154,6 @@ class MOVAPipelineConfig(PipelineConfig):
latent_T = (num_samples + audio_vae.hop_length - 1) // audio_vae.hop_length
return (batch_size, audio_vae.latent_dim, latent_T)
def normalize_video_latents(self, latents: torch.Tensor, video_vae) -> torch.Tensor:
latents_mean = getattr(video_vae.config, "latents_mean", None)
latents_std = getattr(video_vae.config, "latents_std", None)
if latents_mean is None or latents_std is None:
return latents
mean = torch.tensor(
latents_mean, device=latents.device, dtype=latents.dtype
).view(1, video_vae.config.z_dim, 1, 1, 1)
inv_std = (
1.0 / torch.tensor(latents_std, device=latents.device, dtype=latents.dtype)
).view(1, video_vae.config.z_dim, 1, 1, 1)
return (latents - mean) * inv_std
def denormalize_video_latents(
self, latents: torch.Tensor, video_vae
) -> torch.Tensor:
@@ -12,11 +12,15 @@ from sglang.multimodal_gen.configs.models.dits.qwenimage import (
QwenImageEditPlus_2511_DitConfig,
)
from sglang.multimodal_gen.configs.models.encoders.qwen_image import Qwen2_5VLConfig
from sglang.multimodal_gen.configs.models.vaes.base import (
get_channelwise_decode_scale_and_shift,
)
from sglang.multimodal_gen.configs.models.vaes.qwenimage import QwenImageVAEConfig
from sglang.multimodal_gen.configs.pipeline_configs.base import (
ImagePipelineConfig,
ModelTaskType,
maybe_unpad_latents,
pack_latents_2x2,
pad_text_embeddings_with_mask,
shard_rotary_emb_for_sp,
)
@@ -141,19 +145,6 @@ def _shard_qwen_edit_freqs_cis_for_sp(freqs_cis, noisy_img_seq_len, device):
)
# Copied from diffusers.pipelines.qwenimage.pipeline_qwenimage.QwenImagePipeline._pack_latents
def _pack_latents(latents, batch_size, num_channels_latents, height, width):
latents = latents.view(
batch_size, num_channels_latents, height // 2, 2, width // 2, 2
)
latents = latents.permute(0, 2, 4, 1, 3, 5)
latents = latents.reshape(
batch_size, (height // 2) * (width // 2), num_channels_latents * 4
)
return latents
@dataclass
class QwenImagePipelineConfig(QwenImageRolloutPipelineMixin, ImagePipelineConfig):
"""Configuration for the QwenImage pipeline."""
@@ -292,19 +283,14 @@ class QwenImagePipelineConfig(QwenImageRolloutPipelineMixin, ImagePipelineConfig
width = 2 * (batch.width // (vae_scale_factor * 2))
num_channels_latents = self.dit_config.arch_config.in_channels // 4
# pack latents
return _pack_latents(latents, batch_size, num_channels_latents, height, width)
return pack_latents_2x2(
latents, batch_size, num_channels_latents, height, width
)
def get_decode_scale_and_shift(self, device, dtype, vae):
vae_arch_config = self.vae_config.arch_config
scaling_factor = 1.0 / torch.tensor(
vae_arch_config.latents_std, device=device
).view(1, vae_arch_config.z_dim, 1, 1, 1).to(device, dtype)
shift_factor = (
torch.tensor(vae_arch_config.latents_mean)
.view(1, vae_arch_config.z_dim, 1, 1, 1)
.to(device, dtype)
return get_channelwise_decode_scale_and_shift(
self.vae_config.arch_config, device, dtype
)
return scaling_factor, shift_factor
@staticmethod
def get_freqs_cis(img_shapes, txt_seq_lens, rotary_emb, device, dtype):
@@ -594,7 +580,7 @@ class QwenImageEditPipelineConfig(QwenImagePipelineConfig):
image_latents = latent_condition
image_latent_height, image_latent_width = image_latents.shape[3:]
num_channels_latents = self.dit_config.arch_config.in_channels // 4
image_latents = _pack_latents(
image_latents = pack_latents_2x2(
image_latents,
batch_size,
num_channels_latents,
@@ -639,21 +625,6 @@ VAE_IMAGE_SIZE = 1024 * 1024
class QwenImageEditPlusPipelineConfig(QwenImageEditPipelineConfig):
task_type: ModelTaskType = ModelTaskType.I2I
def _get_condition_image_sizes(self, batch) -> list[tuple[int, int]]:
image = batch.condition_image
if not isinstance(image, list):
image = [image]
condition_image_sizes = []
for img in image:
image_width, image_height = img.size
edit_width, edit_height, _ = _calculate_dimensions(
VAE_IMAGE_SIZE, image_width / image_height
)
condition_image_sizes.append((edit_width, edit_height))
return condition_image_sizes
def prepare_image_processor_kwargs(self, batch, neg=False) -> dict:
prompt = batch.prompt if not neg else batch.negative_prompt
if not prompt:
@@ -686,14 +657,6 @@ class QwenImageEditPlusPipelineConfig(QwenImageEditPipelineConfig):
def prepare_calculated_size(self, image):
return self.calculate_vae_image_size(image, image.width, image.height)
def resize_condition_image(self, images, target_width, target_height):
if not isinstance(images, list):
images = [images]
new_images = []
for img, width, height in zip(images, target_width, target_height):
new_images.append(resize(img, height, width, resize_mode="default"))
return new_images
def calculate_condition_image_size(self, image, width, height) -> tuple[int, int]:
calculated_width, calculated_height, _ = _calculate_dimensions(
CONDITION_IMAGE_SIZE, width / height
@@ -11,7 +11,10 @@ from sglang.multimodal_gen.configs.models.dits.sana_wm import SanaWMConfig
from sglang.multimodal_gen.configs.models.encoders import BaseEncoderOutput
from sglang.multimodal_gen.configs.models.encoders.base import EncoderConfig
from sglang.multimodal_gen.configs.models.encoders.gemma2 import Gemma2Config
from sglang.multimodal_gen.configs.models.vaes.ltx_video import LTXVideoVAEConfig
from sglang.multimodal_gen.configs.models.vaes.ltx_video import (
LTXVideoVAEConfig,
get_ltx_video_decode_scale_and_shift,
)
from sglang.multimodal_gen.configs.pipeline_configs.base import (
ModelTaskType,
PipelineConfig,
@@ -286,42 +289,9 @@ class SanaWMPipelineConfig(PipelineConfig):
return latents
def get_decode_scale_and_shift(self, device, dtype, vae):
"""Invert the LTX-2 latent normalization used before denoising.
SANA-WM uses the LTX-2 VAE. Upstream encodes as
``(z - latents_mean) * scaling_factor / latents_std`` and decodes by
applying the inverse transform.
"""
latents_mean = getattr(vae, "latents_mean", None)
latents_std = getattr(vae, "latents_std", None)
scaling_factor = (
getattr(getattr(vae, "config", None), "scaling_factor", None)
or getattr(vae, "scaling_factor", None)
or getattr(self.vae_config.arch_config, "scaling_factor", None)
or 1.0
return get_ltx_video_decode_scale_and_shift(
device, dtype, vae, self.vae_config.arch_config
)
if isinstance(scaling_factor, (int, float)) and float(scaling_factor) == 0.0:
scaling_factor = 1.0
if isinstance(latents_mean, torch.Tensor) and isinstance(
latents_std, torch.Tensor
):
latents_mean = latents_mean.to(device=device, dtype=dtype).view(
1, -1, 1, 1, 1
)
latents_std = latents_std.to(device=device, dtype=dtype).view(
1, -1, 1, 1, 1
)
sf = torch.tensor(float(scaling_factor), device=device, dtype=dtype).view(
1, 1, 1, 1, 1
)
return sf / latents_std, latents_mean
sf = torch.tensor(float(scaling_factor), device=device, dtype=dtype).view(
1, 1, 1, 1, 1
)
return sf, None
class SanaWMRealtimeConfig(SanaWMPipelineConfig):
@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Any
from sglang.multimodal_gen.configs.sample.sampling_params import (
DataType,
SamplingParams,
_sanitize_filename,
)
from sglang.multimodal_gen.configs.task_type import ModelTaskType
@@ -65,15 +66,7 @@ class ActionSamplingParams:
if env_steps is not None and self.num_inference_steps is not None:
self.num_inference_steps = int(env_steps)
def build_request_extra(self) -> dict[str, Any]:
extra = {}
diffusers_kwargs = getattr(self, "diffusers_kwargs", None)
if diffusers_kwargs:
extra["diffusers_kwargs"] = diffusers_kwargs
explicit_fields = getattr(self, "_explicit_fields", None)
if explicit_fields is not None:
extra["explicit_fields"] = sorted(explicit_fields)
return extra
build_request_extra = SamplingParams.build_request_extra
def apply_request_extra(self, req: Any) -> None:
req.extra.update(self.build_request_extra())
@@ -1,5 +1,4 @@
# SPDX-License-Identifier: Apache-2.0
import math
import os
from collections.abc import Mapping
from dataclasses import dataclass, field
@@ -8,6 +7,7 @@ from typing import Any, ClassVar
import msgspec
from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
from sglang.multimodal_gen.configs.utils import optional_positive_finite_float
_MINIMAX_H3_MAX_SIGNED_SEED = (1 << 63) - 1
@@ -23,17 +23,6 @@ def _optional_unit_float(value: Any, field_name: str) -> float | None:
return out
def _optional_positive_finite_float(value: Any, field_name: str) -> float | None:
if value is None:
return None
if isinstance(value, bool) or not isinstance(value, (int, float)):
raise ValueError(f"{field_name} must be a number")
out = float(value)
if not math.isfinite(out) or out <= 0.0:
raise ValueError(f"{field_name} must be a positive finite number")
return out
@dataclass
class MiniMaxH3SamplingParams(SamplingParams):
# The video/audio sigma schedules include both interval endpoints.
@@ -226,8 +215,8 @@ class MiniMaxH3SamplingParams(SamplingParams):
if field_name in self.target
}
super()._validate()
_optional_positive_finite_float(self.flow_shift, "flow_shift")
_optional_positive_finite_float(self.audio_flow_shift, "audio_flow_shift")
optional_positive_finite_float(self.flow_shift, "flow_shift")
optional_positive_finite_float(self.audio_flow_shift, "audio_flow_shift")
if self.enable_frame_interpolation:
raise ValueError(
"MiniMax H3 does not support enable_frame_interpolation: the "
@@ -758,10 +758,6 @@ class SamplingParams:
RLRolloutArgs.validate_sampling_params(self)
def check_sampling_param(self):
# Keep backward-compatibility for old call sites.
self._validate()
def _validate_with_pipeline_config(self, pipeline_config):
"""
check if the sampling params is compatible and valid with server_args
+12 -13
View File
@@ -1,12 +1,23 @@
# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo
import argparse
import math
import os
from dataclasses import fields
from operator import attrgetter
from typing import Any
def optional_positive_finite_float(value: Any, field_name: str) -> float | None:
if value is None:
return None
if isinstance(value, bool) or not isinstance(value, (int, float)):
raise ValueError(f"{field_name} must be a number")
out = float(value)
if not math.isfinite(out) or out <= 0.0:
raise ValueError(f"{field_name} must be a positive finite number")
return out
def expand_path_fields(obj) -> None:
"""Expand paths in dataclass configuration without modifying other fields."""
for f in fields(obj):
@@ -81,15 +92,3 @@ def update_config_from_args(
args_dict.pop(key)
return len(args_to_remove) > 0
def clean_cli_args(args: argparse.Namespace) -> dict[str, Any]:
"""
Clean the arguments by removing the ones that not explicitly provided by the user.
"""
provided_args = {}
for k, v in vars(args).items():
if v is not None and hasattr(args, "_provided") and k in args._provided:
provided_args[k] = v
return provided_args
+9 -5
View File
@@ -5,12 +5,21 @@
import logging
import os
import warnings
from typing import TYPE_CHECKING, Any, Callable
from sglang.multimodal_gen.runtime.utils.common import get_bool_env_var
logger = logging.getLogger(__name__)
if "SGLANG_DIFFUSION_TRACE_FUNCTION" in os.environ:
warnings.warn(
"SGLANG_DIFFUSION_TRACE_FUNCTION was unused and has been removed. "
"Use Python profiling tools for function-level tracing.",
FutureWarning,
stacklevel=2,
)
if TYPE_CHECKING:
SGLANG_DIFFUSION_NCCL_SO_PATH: str | None = None
LD_LIBRARY_PATH: str | None = None
@@ -20,7 +29,6 @@ if TYPE_CHECKING:
SGLANG_DIFFUSION_CONFIG_ROOT: str = os.path.expanduser("~/.config/sgl_diffusion")
SGLANG_DIFFUSION_LOGGING_LEVEL: str = "INFO"
SGLANG_DIFFUSION_LOGGING_PREFIX: str = ""
SGLANG_DIFFUSION_TRACE_FUNCTION: int = 0
SGLANG_DIFFUSION_DISABLE_EARLY_VAE_DECODER_CAST: bool = False
SGLANG_DIFFUSION_DISABLE_VAE_DECODER_STORE: bool = False
SGLANG_DIFFUSION_DISABLE_MAPPED_WILLNEED: bool = False
@@ -229,10 +237,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
),
# if set, SGLANG_DIFFUSION_LOGGING_PREFIX will be prepended to all log messages
"SGLANG_DIFFUSION_LOGGING_PREFIX": _lazy_str("SGLANG_DIFFUSION_LOGGING_PREFIX", ""),
# Trace function calls
# If set to 1, sgl_diffusion will trace function calls
# Useful for debugging
"SGLANG_DIFFUSION_TRACE_FUNCTION": _lazy_int("SGLANG_DIFFUSION_TRACE_FUNCTION", 0),
# Path to the attention configuration file. Only used for sliding tile
# attention for now.
"SGLANG_DIFFUSION_ATTENTION_CONFIG": _lazy_path(
@@ -30,16 +30,8 @@ def is_ideogram_transformer(current_model: Any, call_kwargs: dict) -> bool:
)
def _unwrap_model(current_model: Any) -> Any:
for attr in ("module", "_orig_mod"):
wrapped = getattr(current_model, attr, None)
if wrapped is not None:
current_model = wrapped
return current_model
def _dynamic_mask_meta(current_model: Any) -> DynamicVarlenMaskMeta:
model = _unwrap_model(current_model)
model = bcg_utils.unwrap_model(current_model)
meta = getattr(model, _DYNAMIC_MASK_META_ATTR, None)
if not isinstance(meta, DynamicVarlenMaskMeta):
meta = DynamicVarlenMaskMeta()
@@ -55,16 +55,8 @@ def _pad_caption(obj: Any, *, target: int) -> Any:
return obj
def _unwrap_model(current_model: Any) -> Any:
for attr in ("module", "_orig_mod"):
wrapped = getattr(current_model, attr, None)
if wrapped is not None:
current_model = wrapped
return current_model
def _build_caption_freqs(current_model: Any, *, target: int, device: torch.device):
rotary_emb = getattr(_unwrap_model(current_model), "rotary_emb", None)
rotary_emb = getattr(bcg_utils.unwrap_model(current_model), "rotary_emb", None)
if rotary_emb is None:
return None
@@ -249,6 +249,14 @@ def pad_masked_prompt_kwargs(call_kwargs: dict, buckets: tuple[int, ...]) -> dic
return out
def unwrap_model(current_model: Any) -> Any:
for attr in ("module", "_orig_mod"):
wrapped = getattr(current_model, attr, None)
if wrapped is not None:
current_model = wrapped
return current_model
def transformer_class_name_matches(current_model: Any, needle: str) -> bool:
"""True when ``current_model`` (or its ``module`` / ``_orig_mod`` wrapper)
is a transformer whose qualified class name contains ``needle``."""
@@ -1,28 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
"""Compatibility shim for disaggregated diffusion argument helpers."""
from __future__ import annotations
import argparse
from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
from sglang.multimodal_gen.runtime.server_args.disagg import DisaggServerArgsMixin
# Keep the historical disagg_args import path working.
DISAGG_RESULT_PORT_OFFSETS = DisaggServerArgsMixin.DISAGG_RESULT_PORT_OFFSETS
DisaggArgsMixin = DisaggServerArgsMixin
def add_disagg_cli_args(parser: argparse.ArgumentParser) -> None:
"""Register disaggregated-diffusion CLI args through ServerArgs."""
from sglang.multimodal_gen.runtime.server_args import ServerArgs
ServerArgs.add_disagg_cli_args(parser)
def convert_disagg_role_string(kwargs: dict) -> None:
"""Convert ``disagg_role`` from string to ``RoleType`` enum in-place."""
if "disagg_role" in kwargs and isinstance(kwargs["disagg_role"], str):
kwargs["disagg_role"] = RoleType.from_string(kwargs["disagg_role"])
@@ -2,11 +2,8 @@
"""Dispatch policies for multi-instance disaggregated diffusion pipelines."""
import abc
import logging
import threading
logger = logging.getLogger(__name__)
class DispatchPolicy(abc.ABC):
def __init__(self, num_instances: int):
@@ -19,16 +16,9 @@ class DispatchPolicy(abc.ABC):
return self._num_instances
@abc.abstractmethod
def select(self, active_counts: list[int] | None = None) -> int: ...
def select_with_capacity(self, free_slots: list[int]) -> int | None:
"""Select an instance that has free capacity, or None if all full."""
if not any(s > 0 for s in free_slots):
return None
return self.select(active_counts=None)
def record_completion(self, instance_id: int) -> None:
pass
...
class RoundRobin(DispatchPolicy):
@@ -37,12 +27,6 @@ class RoundRobin(DispatchPolicy):
self._lock = threading.Lock()
self._next = 0
def select(self, active_counts: list[int] | None = None) -> int:
with self._lock:
chosen = self._next
self._next = (self._next + 1) % self._num_instances
return chosen
def select_with_capacity(self, free_slots: list[int]) -> int | None:
with self._lock:
for _ in range(self._num_instances):
@@ -56,43 +40,11 @@ class RoundRobin(DispatchPolicy):
class MaxFreeSlotsFirst(DispatchPolicy):
"""Dispatch to the instance with the most free slots."""
def __init__(self, num_instances: int, max_slots_per_instance: int = 1):
def __init__(self, num_instances: int):
super().__init__(num_instances)
self._max_slots = max_slots_per_instance
self._lock = threading.Lock()
self._tiebreak = 0
def select(self, active_counts: list[int] | None = None) -> int:
with self._lock:
if active_counts is None or len(active_counts) != self._num_instances:
chosen = self._tiebreak % self._num_instances
self._tiebreak += 1
return chosen
best_id = 0
best_free = self._max_slots - active_counts[0]
for i in range(1, self._num_instances):
free = self._max_slots - active_counts[i]
if free > best_free:
best_free = free
best_id = i
elif free == best_free:
if i == (self._tiebreak % self._num_instances):
best_id = i
self._tiebreak += 1
if best_free <= 0:
logger.warning(
"All %d instances are at capacity (%d slots each), "
"dispatching to instance %d anyway",
self._num_instances,
self._max_slots,
best_id,
)
return best_id
def select_with_capacity(self, free_slots: list[int]) -> int | None:
with self._lock:
best_id = -1
@@ -121,26 +73,10 @@ class PoolDispatcher:
num_denoisers: int,
num_decoders: int,
policy_name: str = "round_robin",
**kwargs,
):
self.encoder_policy = create_dispatch_policy(
policy_name, num_encoders, **kwargs
)
self.denoiser_policy = create_dispatch_policy(
policy_name, num_denoisers, **kwargs
)
self.decoder_policy = create_dispatch_policy(
policy_name, num_decoders, **kwargs
)
def select_encoder(self, active_counts: list[int] | None = None) -> int:
return self.encoder_policy.select(active_counts)
def select_denoiser(self, active_counts: list[int] | None = None) -> int:
return self.denoiser_policy.select(active_counts)
def select_decoder(self, active_counts: list[int] | None = None) -> int:
return self.decoder_policy.select(active_counts)
self.encoder_policy = create_dispatch_policy(policy_name, num_encoders)
self.denoiser_policy = create_dispatch_policy(policy_name, num_denoisers)
self.decoder_policy = create_dispatch_policy(policy_name, num_decoders)
def select_encoder_with_capacity(self, free_slots: list[int]) -> int | None:
return self.encoder_policy.select_with_capacity(free_slots)
@@ -152,7 +88,7 @@ class PoolDispatcher:
return self.decoder_policy.select_with_capacity(free_slots)
def create_dispatch_policy(name: str, num_instances: int, **kwargs) -> DispatchPolicy:
def create_dispatch_policy(name: str, num_instances: int) -> DispatchPolicy:
policies = {
"round_robin": RoundRobin,
"max_free_slots": MaxFreeSlotsFirst,
@@ -162,4 +98,4 @@ def create_dispatch_policy(name: str, num_instances: int, **kwargs) -> DispatchP
raise ValueError(
f"Unknown dispatch policy '{name}'. Available: {list(policies.keys())}"
)
return cls(num_instances=num_instances, **kwargs)
return cls(num_instances=num_instances)
@@ -6,12 +6,6 @@ import time
from dataclasses import dataclass
@dataclass
class _RequestTiming:
start_time: float
stage_start: float = 0.0
@dataclass
class RoleStats:
role: str
@@ -54,7 +48,7 @@ class DisaggMetrics:
self._failed = 0
self._timed_out = 0
self._in_flight: dict[str, _RequestTiming] = {}
self._in_flight: dict[str, float] = {}
self._last_latency = 0.0
self._max_latency = 0.0
@@ -71,14 +65,14 @@ class DisaggMetrics:
def record_request_start(self, request_id: str) -> None:
with self._lock:
self._in_flight[request_id] = _RequestTiming(start_time=time.monotonic())
self._in_flight[request_id] = time.monotonic()
def record_request_complete(self, request_id: str) -> None:
now = time.monotonic()
with self._lock:
timing = self._in_flight.pop(request_id, None)
if timing is not None:
latency = now - timing.start_time
start_time = self._in_flight.pop(request_id, None)
if start_time is not None:
latency = now - start_time
self._last_latency = latency
self._max_latency = max(self._max_latency, latency)
self._total_latency += latency
@@ -865,8 +865,8 @@ class DiffusionServer:
if idx is None:
break
entry = self._denoiser_tta.popleft()
self._transfer_dispatch_to_denoiser(
entry.request_id, entry.transfer_state, idx
self._transfer_dispatch_to_receiver(
entry.request_id, entry.transfer_state, idx, RoleType.DENOISER
)
def _drain_decoder_tta(self) -> None:
@@ -877,8 +877,8 @@ class DiffusionServer:
if idx is None:
break
entry = self._decoder_tta.popleft()
self._transfer_dispatch_to_decoder(
entry.request_id, entry.transfer_state, idx
self._transfer_dispatch_to_receiver(
entry.request_id, entry.transfer_state, idx, RoleType.DECODER
)
def _extract_request_id(self, frames: list) -> str | None:
@@ -1161,36 +1161,57 @@ class DiffusionServer:
)
receiver_pushes[receiver_idx].send_multipart(encode_transfer_msg(alloc_msg))
def _transfer_dispatch_to_denoiser(
self, request_id: str, p2p: _TransferRequestState, denoiser_idx: int
def _transfer_dispatch_to_receiver(
self,
request_id: str,
p2p: _TransferRequestState,
receiver_idx: int,
role: RoleType,
) -> None:
self._denoiser_free_slots[denoiser_idx] -= 1
p2p.receiver_instance = denoiser_idx
if role == RoleType.DENOISER:
free_slots = self._denoiser_free_slots
peers = self._denoiser_peers
sender_pushes = self._encoder_pushes
receiver_pushes = self._denoiser_pushes
source_role = "encoder"
running_state = RequestState.DENOISING_RUNNING
elif role == RoleType.DECODER:
free_slots = self._decoder_free_slots
peers = self._decoder_peers
sender_pushes = self._denoiser_pushes
receiver_pushes = self._decoder_pushes
source_role = "denoiser"
running_state = RequestState.DECODER_RUNNING
else:
raise ValueError(f"Invalid transfer receiver role: {role}")
free_slots[receiver_idx] -= 1
p2p.receiver_instance = receiver_idx
try:
self._tracker.transition(
request_id,
RequestState.DENOISING_RUNNING,
denoiser_instance=denoiser_idx,
running_state,
**{f"{role.value}_instance": receiver_idx},
)
except ValueError:
pass
peer_info = self._denoiser_peers.get(denoiser_idx, {})
peer_info = peers.get(receiver_idx, {})
if not self._try_fast_path_push(
request_id=request_id,
p2p=p2p,
receiver_peer_info=peer_info,
sender_pushes=self._encoder_pushes,
receiver_role_label="denoiser",
receiver_idx=denoiser_idx,
sender_pushes=sender_pushes,
receiver_role_label=role.value,
receiver_idx=receiver_idx,
):
self._send_slow_path_alloc(
request_id=request_id,
p2p=p2p,
receiver_pushes=self._denoiser_pushes,
receiver_idx=denoiser_idx,
source_role="encoder",
receiver_pushes=receiver_pushes,
receiver_idx=receiver_idx,
source_role=source_role,
)
def _handle_transfer_allocated(self, msg: dict) -> None:
@@ -1378,38 +1399,6 @@ class DiffusionServer:
self._transfer_state.pop(request_id, None)
def _transfer_dispatch_to_decoder(
self, request_id: str, p2p: _TransferRequestState, decoder_idx: int
) -> None:
self._decoder_free_slots[decoder_idx] -= 1
p2p.receiver_instance = decoder_idx
try:
self._tracker.transition(
request_id,
RequestState.DECODER_RUNNING,
decoder_instance=decoder_idx,
)
except ValueError:
pass
peer_info = self._decoder_peers.get(decoder_idx, {})
if not self._try_fast_path_push(
request_id=request_id,
p2p=p2p,
receiver_peer_info=peer_info,
sender_pushes=self._denoiser_pushes,
receiver_role_label="decoder",
receiver_idx=decoder_idx,
):
self._send_slow_path_alloc(
request_id=request_id,
p2p=p2p,
receiver_pushes=self._decoder_pushes,
receiver_idx=decoder_idx,
source_role="denoiser",
)
def _transfer_return_to_client_from_msg(self, request_id: str, msg: dict) -> None:
with self._lock:
client_identity = self._pending.pop(request_id, None)
@@ -427,9 +427,7 @@ class SchedulerDisaggMixin:
from sglang.multimodal_gen.runtime.disaggregation.metrics import DisaggMetrics
self._disagg_role = server_args.disagg_role
self._disagg_timeout_s = float(getattr(server_args, "disagg_timeout", 600))
self._disagg_metrics = None
self._disagg_mode = getattr(server_args, "disagg_mode", False)
self._pool_work_pull = None
self._pool_result_push = None
self._transfer_manager = None
@@ -45,11 +45,6 @@ def sequence_model_parallel_all_gather(
return get_sp_group().all_gather(input_, dim)
def sequence_model_parallel_all_reduce(input_: torch.Tensor) -> torch.Tensor:
"""All-reduce the input tensor across model parallel group."""
return get_sp_group().all_reduce(input_)
def cfg_model_parallel_all_gather(
input_: torch.Tensor, dim: int = -1, separate_tensors: bool = False
) -> torch.Tensor:
@@ -270,7 +270,6 @@ class DeviceCommunicatorBase:
self.world_size = dist.get_world_size(cpu_group)
self.ranks = dist.get_process_group_ranks(cpu_group)
self.global_rank = dist.get_rank()
self.global_world_size = dist.get_world_size()
self.rank_in_group = dist.get_group_rank(self.cpu_group, self.global_rank)
def all_reduce(
@@ -169,7 +169,6 @@ class GroupCoordinator:
rank_in_group: int # rank inside the group
cpu_group: ProcessGroup # group for CPU communication
device_group: ProcessGroup # group for device communication
use_device_communicator: bool # whether to use device communicator
device_communicator: DeviceCommunicatorBase # all_to_all_4D communicator
def __init__(
@@ -207,7 +206,6 @@ class GroupCoordinator:
# TODO: fix it for other platforms
self.device = get_local_torch_device()
self.use_device_communicator = use_device_communicator
self.device_communicator: DeviceCommunicatorBase = None # type: ignore
if use_device_communicator and self.world_size > 1:
communicator_cls = _resolve_all_to_all_communicator_cls()
@@ -228,10 +226,6 @@ class GroupCoordinator:
# srt owns topology, dtype, contiguity, and size dispatch for custom ar
self._init_srt_custom_allreduce()
# TODO(will): check if this is needed
# self.use_custom_op_call = current_platform.is_cuda_alike()
self.use_custom_op_call = False
def _init_srt_custom_allreduce(self) -> None:
custom_allreduce_kwargs = {
"group": self.cpu_group,
@@ -767,11 +767,6 @@ def get_sequence_parallel_world_size() -> int:
return get_sp_world_size()
def get_sequence_parallel_rank() -> int:
"""Return my rank for the sequence parallel group."""
return get_sp_parallel_rank()
def get_ulysses_parallel_world_size() -> int:
return get_sp_group().ulysses_world_size
@@ -803,32 +798,6 @@ def get_ring_ctx() -> tuple[int, int]:
return get_ring_parallel_world_size(), get_ring_parallel_rank()
# PP
def get_pp_group() -> GroupCoordinator:
assert _PP is not None, "pipeline model parallel group is not initialized"
return _PP
def get_pipeline_parallel_world_size() -> int:
"""Return world size for the pipeline model parallel group."""
return get_pp_group().world_size
def get_pipeline_parallel_rank() -> int:
"""Return my rank for the pipeline model parallel group."""
return get_pp_group().rank_in_group
def is_pipeline_first_stage() -> bool:
"""Return True if in the first pipeline model parallel stage, False otherwise."""
return get_pipeline_parallel_rank() == 0
def is_pipeline_last_stage() -> bool:
"""Return True if in the last pipeline model parallel stage, False otherwise."""
return get_pipeline_parallel_rank() == (get_pipeline_parallel_world_size() - 1)
# CFG
def get_cfg_group() -> GroupCoordinator:
assert _CFG is not None, (
@@ -852,47 +821,6 @@ def get_data_parallel_world_size() -> int:
return get_dp_world_size()
def get_data_parallel_rank() -> int:
"""Return my rank for the data parallel group."""
return get_dp_rank()
def is_dp_last_group() -> bool:
"""Return True if in the last data parallel group, False otherwise."""
return (
get_sequence_parallel_rank() == (get_sequence_parallel_world_size() - 1)
and get_classifier_free_guidance_rank()
== (get_classifier_free_guidance_world_size() - 1)
and get_pipeline_parallel_rank() == (get_pipeline_parallel_world_size() - 1)
)
def get_dit_world_size() -> int:
"""Return world size for the DiT model (excluding VAE)."""
return (
get_data_parallel_world_size()
* get_classifier_free_guidance_world_size()
* get_sequence_parallel_world_size()
* get_pipeline_parallel_world_size()
* get_tensor_model_parallel_world_size()
)
def get_vae_parallel_group() -> ProcessGroup:
assert _VAE is not None, "VAE parallel group is not initialized"
return _VAE
def get_vae_parallel_world_size() -> int:
"""Return world size for the VAE parallel group."""
return torch.distributed.get_world_size(group=get_vae_parallel_group())
def get_vae_parallel_rank() -> int:
"""Return my rank for the VAE parallel group."""
return torch.distributed.get_rank(group=get_vae_parallel_group())
def get_decode_parallel_group_coordinator() -> GroupCoordinator:
assert _VAE_DECODE is not None, "VAE decode parallel group is not initialized"
return _VAE_DECODE
@@ -215,16 +215,6 @@ class GenerateSubcommand(CLISubcommand):
def __init__(self) -> None:
self.name = "generate"
super().__init__()
self.init_arg_names = self._get_init_arg_names()
self.generation_arg_names = self._get_generation_arg_names()
def _get_init_arg_names(self) -> list[str]:
"""Get names of arguments for DiffGenerator initialization"""
return ["num_gpus", "tp_size", "sp_size", "model_path"]
def _get_generation_arg_names(self) -> list[str]:
"""Get names of arguments for generate_video method"""
return [field.name for field in dataclasses.fields(SamplingParams)]
def cmd(
self, args: argparse.Namespace, unknown_args: list[str] | None = None
@@ -173,10 +173,6 @@ class VideoListResponse(BaseModel):
object: str = "list"
class VideoRemixRequest(BaseModel):
prompt: str
class RealtimeVideoGenerationsRequest(VideoGenerationsRequest):
enhance_prompt: Literal[False] = False
type: Literal["init"]
@@ -38,7 +38,6 @@ if TYPE_CHECKING:
SANA_WM_DEFAULT_SIZE = "1280x704"
SANA_WM_DEFAULT_NUM_FRAMES = 1081
SANA_WM_DEFAULT_FPS = 16
SANA_WM_DEFAULT_STEPS = 4
SANA_WM_DEFAULT_GUIDANCE = 1.0
@@ -41,9 +41,6 @@ class SiluAndMul(CustomOp):
return: (num_tokens, d) or (batch_size, seq_len, d)
"""
def __init__(self) -> None:
super().__init__()
def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
d = x.shape[-1] // 2
output_shape = x.shape[:-1] + (d,)
@@ -111,9 +108,6 @@ class GeluAndMul(CustomOp):
@CustomOp.register("gelu_new")
class NewGELU(CustomOp):
def __init__(self):
super().__init__()
def forward_cuda(self, *args, **kwargs) -> Any:
return self.forward_native(*args, **kwargs)
@@ -129,9 +123,6 @@ class NewGELU(CustomOp):
@CustomOp.register("quick_gelu")
class QuickGELU(CustomOp):
# https://github.com/huggingface/transformers/blob/main/src/transformers/activations.py#L90
def __init__(self):
super().__init__()
def forward_cuda(self, *args, **kwargs) -> Any:
return self.forward_native(*args, **kwargs)
@@ -196,12 +196,6 @@ class AscendFAMetadata:
class AscendFAMetadataBuilder(AttentionMetadataBuilder):
def __init__(self) -> None:
pass
def prepare(self) -> None:
pass
def build(
self,
**kwargs: dict[str, Any],
@@ -95,16 +95,6 @@ T = TypeVar("T", bound=AttentionMetadata)
class AttentionMetadataBuilder(ABC, Generic[T]):
"""Abstract class for attention metadata builders."""
@abstractmethod
def __init__(self) -> None:
"""Create the builder, remember some configuration and parameters."""
raise NotImplementedError
@abstractmethod
def prepare(self) -> None:
"""Prepare for one batch."""
raise NotImplementedError
@abstractmethod
def build(
self,
@@ -53,12 +53,6 @@ class BlockSparseAttentionMetadata(AttentionMetadata):
class BlockSparseAttentionMetadataBuilder(AttentionMetadataBuilder):
def __init__(self) -> None:
pass
def prepare(self) -> None:
pass
def build(
self,
current_timestep: int,
@@ -62,12 +62,6 @@ class CubeSparseAttentionMetadata(AttentionMetadata):
class CubeSparseAttentionMetadataBuilder(AttentionMetadataBuilder):
def __init__(self):
pass
def prepare(self):
pass
def build( # type: ignore[override]
self,
*,
@@ -322,12 +322,6 @@ class FlashAttentionMetadata:
class FlashAttentionMetadataBuilder(AttentionMetadataBuilder):
def __init__(self) -> None:
pass
def prepare(self) -> None:
pass
def build( # type: ignore
self,
raw_latent_shape=list,
@@ -386,7 +380,6 @@ class FlashAttentionImpl(AttentionImpl):
self.packed_trailing_padding = extra_impl_args.get(
"packed_trailing_padding", False
)
self.attention_metadata = FlashAttentionMetadata()
def _request_skip_softmax_threshold(self) -> tuple[bool, float | None]:
params = get_request_skip_softmax_params()
@@ -268,12 +268,6 @@ class HybridWindowAttentionH3Metadata(AttentionMetadata):
class HybridWindowAttentionH3MetadataBuilder(AttentionMetadataBuilder):
def __init__(self) -> None:
pass
def prepare(self) -> None:
pass
def build( # type: ignore[override]
self,
*,
@@ -54,12 +54,6 @@ class RainFusionAttentionMetadata(AttentionMetadata):
class RainFusionAttentionMetadataBuilder(AttentionMetadataBuilder):
def __init__(self) -> None:
pass
def prepare(self) -> None:
pass
def build(
self,
current_timestep: int,
@@ -80,12 +80,6 @@ class SlidingTileAttentionMetadata(AttentionMetadata):
class SlidingTileAttentionMetadataBuilder(AttentionMetadataBuilder):
def __init__(self):
pass
def prepare(self):
pass
def build( # type: ignore
self,
STA_param: list[list[Any]],
@@ -86,12 +86,6 @@ class SparseLinearAttentionMetadata(AttentionMetadata):
class SparseLinearAttentionMetadataBuilder(AttentionMetadataBuilder):
"""Builder for SparseLinearAttentionMetadata."""
def __init__(self) -> None:
pass
def prepare(self) -> None:
pass
def build(
self,
current_timestep: int,
@@ -104,6 +98,24 @@ class SparseLinearAttentionMetadataBuilder(AttentionMetadataBuilder):
)
def _initialize_linear_attention(model, head_size: int, feature_map: str) -> None:
model.proj_l = nn.Linear(head_size, head_size, dtype=torch.float32)
model.feature_map_q: Callable[[torch.Tensor], torch.Tensor]
model.feature_map_k: Callable[[torch.Tensor], torch.Tensor]
if feature_map == "elu":
model.feature_map_q = lambda x: F.elu(x) + 1
model.feature_map_k = lambda x: F.elu(x) + 1
elif feature_map == "relu":
model.feature_map_q = F.relu
model.feature_map_k = F.relu
elif feature_map == "softmax":
model.feature_map_q = lambda x: F.softmax(x, dim=-1)
model.feature_map_k = lambda x: F.softmax(x, dim=-1)
else:
raise ValueError(f"Unknown feature map: {feature_map}")
model._init_weights()
class SparseLinearAttentionImpl(AttentionImpl, nn.Module):
"""Implementation of sparse linear attention for the backend."""
@@ -131,26 +143,7 @@ class SparseLinearAttentionImpl(AttentionImpl, nn.Module):
self.BLKK = BLKK
self.dtype = torch.bfloat16 if use_bf16 else torch.float16
# Learnable linear projection for combining sparse + linear attention
self.proj_l = nn.Linear(head_size, head_size, dtype=torch.float32)
# Feature map for linear attention
# Type annotation for callables
self.feature_map_q: Callable[[torch.Tensor], torch.Tensor]
self.feature_map_k: Callable[[torch.Tensor], torch.Tensor]
if feature_map == "elu":
self.feature_map_q = lambda x: F.elu(x) + 1
self.feature_map_k = lambda x: F.elu(x) + 1
elif feature_map == "relu":
self.feature_map_q = F.relu
self.feature_map_k = F.relu
elif feature_map == "softmax":
self.feature_map_q = lambda x: F.softmax(x, dim=-1)
self.feature_map_k = lambda x: F.softmax(x, dim=-1)
else:
raise ValueError(f"Unknown feature map: {feature_map}")
self._init_weights()
_initialize_linear_attention(self, head_size, feature_map)
def _init_weights(self) -> None:
"""Initialize projection weights to zero for residual-like behavior."""
@@ -322,12 +315,6 @@ class SageSparseLinearAttentionMetadata(AttentionMetadata):
class SageSparseLinearAttentionMetadataBuilder(AttentionMetadataBuilder):
"""Builder for SageSparseLinearAttentionMetadata."""
def __init__(self) -> None:
pass
def prepare(self) -> None:
pass
def build(
self,
current_timestep: int,
@@ -369,42 +356,12 @@ class SageSparseLinearAttentionImpl(AttentionImpl, nn.Module):
self.topk_ratio = topk_ratio
self.dtype = torch.bfloat16 if use_bf16 else torch.float16
# Learnable linear projection for combining sparse + linear attention
self.proj_l = nn.Linear(head_size, head_size, dtype=torch.float32)
_initialize_linear_attention(self, head_size, feature_map)
# Feature map for linear attention
# Type annotation for callables
self.feature_map_q: Callable[[torch.Tensor], torch.Tensor]
self.feature_map_k: Callable[[torch.Tensor], torch.Tensor]
if feature_map == "elu":
self.feature_map_q = lambda x: F.elu(x) + 1
self.feature_map_k = lambda x: F.elu(x) + 1
elif feature_map == "relu":
self.feature_map_q = F.relu
self.feature_map_k = F.relu
elif feature_map == "softmax":
self.feature_map_q = lambda x: F.softmax(x, dim=-1)
self.feature_map_k = lambda x: F.softmax(x, dim=-1)
else:
raise ValueError(f"Unknown feature map: {feature_map}")
self._init_weights()
def _init_weights(self) -> None:
"""Initialize projection weights to zero for residual-like behavior."""
with torch.no_grad():
nn.init.zeros_(self.proj_l.weight)
nn.init.zeros_(self.proj_l.bias) # type: ignore[arg-type]
def _calc_linear_attention_with_torch(
self,
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
):
kv = torch.matmul(k.transpose(-1, -2), v)
k_sum = torch.sum(k, dim=-2, keepdim=True)
return torch.matmul(q, kv) / (1e-5 + torch.matmul(q, k_sum.transpose(-1, -2)))
_init_weights = SparseLinearAttentionImpl._init_weights
_calc_linear_attention_with_torch = (
SparseLinearAttentionImpl._calc_linear_attention_with_torch
)
def forward(
self,
@@ -106,21 +106,7 @@ class SparseVideoGen2AttentionMetadata(AttentionMetadata):
max_seqlen_k: int | None = None
def _require_kwarg(kwargs: dict[str, Any], name: str) -> Any:
if name not in kwargs:
raise ValueError(
f"Missing required argument for SparseVideoGen2Attention: {name}"
)
return kwargs[name]
class SparseVideoGen2AttentionMetadataBuilder(AttentionMetadataBuilder):
def __init__(self) -> None:
pass
def prepare(self) -> None:
pass
def build( # type: ignore[override]
self,
current_timestep: int,
@@ -397,14 +397,6 @@ class SubBlockSparseAttentionMetadata(AttentionMetadata):
class SubBlockSparseAttentionMetadataBuilder(AttentionMetadataBuilder):
# The base class declares __init__ abstract, so a builder that does not
# override it cannot be instantiated at all.
def __init__(self) -> None:
pass
def prepare(self) -> None:
pass
def build( # type: ignore[override]
self, current_timestep: int, **kwargs: dict[str, Any]
) -> SubBlockSparseAttentionMetadata:
@@ -174,12 +174,6 @@ def _compute_cur_topk(attn_metadata: VideoSparseAttentionMetadata) -> int:
class VideoSparseAttentionMetadataBuilder(AttentionMetadataBuilder):
def __init__(self):
pass
def prepare(self):
pass
def build( # type: ignore
self,
current_timestep: int,
@@ -189,7 +183,6 @@ class VideoSparseAttentionMetadataBuilder(AttentionMetadataBuilder):
device: torch.device,
**kwargs: dict[str, Any],
) -> VideoSparseAttentionMetadata:
patch_size = patch_size
dit_seq_shape = (
raw_latent_shape[0] // patch_size[0],
raw_latent_shape[1] // patch_size[1],
@@ -175,9 +175,6 @@ class VideoSparseAttentionH3MetadataBuilder(AttentionMetadataBuilder):
def __init__(self) -> None:
self._workspace_cache: dict = {}
def prepare(self) -> None:
pass
def build( # type: ignore[override]
self,
current_timestep: int,
@@ -88,12 +88,6 @@ def pad_input(hidden_states, indices, batch, seqlen):
class VideoMobaAttentionMetadataBuilder(AttentionMetadataBuilder):
def __init__(self):
pass
def prepare(self):
pass
def build( # type: ignore
self,
current_timestep: int,
@@ -129,14 +129,6 @@ class CustomOp(nn.Module):
# since we are not using Inductor, we always return True
return True
@staticmethod
def default_on() -> bool:
"""
On by default if level < CompilationLevel.PIECEWISE
Specifying 'all' or 'none' in custom_op takes precedence.
"""
raise NotImplementedError
# Dictionary of all custom ops (classes, indexed by registered name).
# To check if an op with a name is enabled, call .enabled() on the class.
# Examples:
@@ -88,6 +88,27 @@ def adjust_scalar_to_fused_array(
return param[shard_id], loaded_weight
def _load_tensor_parallel_weight(
param: Parameter,
loaded_weight: torch.Tensor,
tp_rank: int,
shard_dim: int | None,
) -> None:
is_sharded_weight = getattr(param, "is_sharded_weight", False)
param_data = param.data
# Already-sharded checkpoints (e.g. bitsandbytes) must not be sliced twice.
if shard_dim is not None and not is_sharded_weight:
shard_size = param_data.shape[shard_dim]
loaded_weight = loaded_weight.narrow(
shard_dim, tp_rank * shard_size, shard_size
)
# AutoFP8 scales can be stored as scalars rather than length-one tensors.
if len(loaded_weight.shape) == 0:
loaded_weight = loaded_weight.reshape(1)
assert param_data.shape == loaded_weight.shape
param_data.copy_(loaded_weight)
class LinearMethodBase(QuantizeMethodBase):
"""Base class for different (maybe quantized) linear methods."""
@@ -477,25 +498,9 @@ class ColumnParallelLinear(LinearBase):
self.register_parameter("bias", None)
def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor) -> None:
tp_rank = self.tp_rank
output_dim = getattr(param, "output_dim", None)
is_sharded_weight = getattr(param, "is_sharded_weight", False)
is_sharded_weight = is_sharded_weight
param_data = param.data
if output_dim is not None and not is_sharded_weight:
shard_size = param_data.shape[output_dim]
start_idx = tp_rank * shard_size
loaded_weight = loaded_weight.narrow(output_dim, start_idx, shard_size)
# Special case for loading scales off disk, which often do not
# have a shape (such as in the case of AutoFP8).
if len(loaded_weight.shape) == 0:
loaded_weight = loaded_weight.reshape(1)
assert param_data.shape == loaded_weight.shape
param_data.copy_(loaded_weight)
_load_tensor_parallel_weight(
param, loaded_weight, self.tp_rank, getattr(param, "output_dim", None)
)
def weight_loader_v2(self, param: Parameter, loaded_weight: torch.Tensor) -> None:
# Special case for loading scales off disk, which often do not
@@ -629,7 +634,6 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
is_sharded_weight = getattr(param, "is_sharded_weight", False)
# bitsandbytes loads the weights of the specific portion
# no need to narrow
is_sharded_weight = is_sharded_weight
param_data = param_data.narrow(output_dim, shard_offset, shard_size)
start_idx = tp_rank * shard_size
@@ -1027,7 +1031,6 @@ class QKVParallelLinear(ColumnParallelLinear):
is_sharded_weight = getattr(param, "is_sharded_weight", False)
# bitsandbytes loads the weights of the specific portion
# no need to narrow
is_sharded_weight = is_sharded_weight
shard_idx = 0
param_data = param_data.narrow(output_dim, shard_offset, shard_size)
@@ -1151,26 +1154,9 @@ class RowParallelLinear(LinearBase):
self.register_parameter("bias", None)
def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor):
tp_rank = self.tp_rank
input_dim = getattr(param, "input_dim", None)
is_sharded_weight = getattr(param, "is_sharded_weight", False)
# bitsandbytes loads the weights of the specific portion
# no need to narrow
is_sharded_weight = is_sharded_weight
param_data = param.data
if input_dim is not None and not is_sharded_weight:
shard_size = param_data.shape[input_dim]
start_idx = tp_rank * shard_size
loaded_weight = loaded_weight.narrow(input_dim, start_idx, shard_size)
# Special case for loading scales off disk, which often do not
# have a shape (such as in the case of AutoFP8).
if len(loaded_weight.shape) == 0:
loaded_weight = loaded_weight.reshape(1)
assert param_data.shape == loaded_weight.shape
param_data.copy_(loaded_weight)
_load_tensor_parallel_weight(
param, loaded_weight, self.tp_rank, getattr(param, "input_dim", None)
)
def weight_loader_v2(self, param: BasevLLMParameter, loaded_weight: torch.Tensor):
# Special case for loading scales off disk, which often do not
@@ -32,9 +32,6 @@ from sglang.multimodal_gen.runtime.layers.linear import (
from sglang.multimodal_gen.runtime.layers.quantization.weight_only_fp8 import (
WeightOnlyFP8Linear,
)
from sglang.multimodal_gen.runtime.layers.vocab_parallel_embedding import (
VocabParallelEmbedding,
)
from sglang.multimodal_gen.runtime.managers.forward_context import (
get_forward_context_or_none,
)
@@ -651,37 +648,7 @@ class BaseLayerWithLoRA(nn.Module):
self.strength = 1.0
class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
"""
Vocab parallel embedding layer with support for LoRA (Low-Rank Adaptation).
Note: The current version does not yet implement the LoRA functionality.
This class behaves exactly the same as the base VocabParallelEmbedding.
Future versions will integrate LoRA functionality to support efficient parameter fine-tuning.
"""
def __init__(
self,
base_layer: VocabParallelEmbedding,
) -> None:
super().__init__(base_layer)
def forward(self, input_: torch.Tensor) -> torch.Tensor:
raise NotImplementedError(
"We don't support VocabParallelEmbeddingWithLoRA yet."
)
class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
def __init__(
self,
base_layer: ColumnParallelLinear,
lora_rank: int | None = None,
lora_alpha: int | None = None,
snapshot_base: bool = True,
) -> None:
super().__init__(base_layer, lora_rank, lora_alpha, snapshot_base)
def forward(self, input_: torch.Tensor) -> torch.Tensor:
if self.disable_lora or (self.merged and not self.has_lora_output_offset):
return self.base_layer(input_)
@@ -738,15 +705,6 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
def __init__(
self,
base_layer: MergedColumnParallelLinear,
lora_rank: int | None = None,
lora_alpha: int | None = None,
snapshot_base: bool = True,
) -> None:
super().__init__(base_layer, lora_rank, lora_alpha, snapshot_base)
def slice_lora_b_weights(self, B: torch.Tensor) -> torch.Tensor:
tp_rank = get_tp_rank()
if B.dim() == 3:
@@ -772,15 +730,6 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
class QKVParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
def __init__(
self,
base_layer: QKVParallelLinear,
lora_rank: int | None = None,
lora_alpha: int | None = None,
snapshot_base: bool = True,
) -> None:
super().__init__(base_layer, lora_rank, lora_alpha, snapshot_base)
def slice_lora_b_weights(
self, B: list[torch.Tensor]
) -> tuple[torch.Tensor, torch.Tensor]:
@@ -802,15 +751,6 @@ class QKVParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
def __init__(
self,
base_layer: RowParallelLinear,
lora_rank: int | None = None,
lora_alpha: int | None = None,
snapshot_base: bool = True,
) -> None:
super().__init__(base_layer, lora_rank, lora_alpha, snapshot_base)
def forward(self, input_: torch.Tensor):
if self.disable_lora or (self.merged and not self.has_lora_output_offset):
return self.base_layer(input_)
@@ -890,15 +830,6 @@ class LinearWithLoRA(BaseLayerWithLoRA):
not a tuple of (output, bias).
"""
def __init__(
self,
base_layer: nn.Linear,
lora_rank: int | None = None,
lora_alpha: int | None = None,
snapshot_base: bool = True,
) -> None:
super().__init__(base_layer, lora_rank, lora_alpha, snapshot_base)
@torch.compile()
def forward(self, x: torch.Tensor) -> torch.Tensor:
lora_A = self.lora_A
@@ -989,7 +920,6 @@ def wrap_with_lora_layer(
"""
supported_layer_types: dict[type[nn.Module], type[BaseLayerWithLoRA]] = {
# the order matters
# VocabParallelEmbedding: VocabParallelEmbeddingWithLoRA,
# Weight-only FP8 LoRA is currently dynamic-only and intended for
# single-GPU deployments.
WeightOnlyFP8Linear: WeightOnlyFP8LinearWithLoRA,
@@ -181,7 +181,6 @@ def split_height_for_parallel_decode(
def _maybe_contiguous_for_sp_gather(x: torch.Tensor) -> torch.Tensor:
if (
x.dim() == 5
and hasattr(torch, "channels_last_3d")
and x.is_contiguous(memory_format=torch.channels_last_3d)
and not x.is_contiguous()
):
@@ -302,7 +301,7 @@ def gather_variable_height(
def _halo_memory_format(reference: torch.Tensor) -> torch.memory_format:
if reference.dim() > 1 and reference.stride(1) == 1:
if reference.dim() == 5 and hasattr(torch, "channels_last_3d"):
if reference.dim() == 5:
return torch.channels_last_3d
if reference.dim() == 4:
return torch.channels_last
@@ -475,7 +474,6 @@ def _conv_preserves_local_height(
def _conv3d_weight_is_channels_last_3d(weight: torch.Tensor) -> bool:
return (
weight.dim() == 5
and hasattr(torch, "channels_last_3d")
and (current_platform.is_cuda() or current_platform.is_rocm())
and weight.is_contiguous(memory_format=torch.channels_last_3d)
)
@@ -21,7 +21,10 @@ import os
import torch
from torch.nn.parameter import Parameter
from sglang.multimodal_gen.runtime.layers.linear import LinearMethodBase
from sglang.multimodal_gen.runtime.layers.linear import (
LinearMethodBase,
UnquantizedLinearMethod,
)
from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import (
ConvRotInt8Config,
)
@@ -118,20 +121,17 @@ class ConvRotInt8ComfyKitchenLinearMethod(LinearMethodBase):
"--quantization-ignored-layers"
)
# The online path initially matches UnquantizedLinearMethod so the
# source weights load in BF16 before quantization. Serialized weights
# allocate their final INT8 storage immediately.
weight = Parameter(
torch.empty(
sum(output_partition_sizes),
input_size_per_partition,
dtype=(torch.int8 if self.is_checkpoint_serialized else params_dtype),
),
requires_grad=False,
# Online checkpoints load their original dtype; serialized ones use INT8.
UnquantizedLinearMethod.create_weights(
self,
layer,
input_size_per_partition,
output_partition_sizes,
input_size,
output_size,
torch.int8 if self.is_checkpoint_serialized else params_dtype,
**extra_weight_attrs,
)
set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0})
layer.register_parameter("weight", weight)
set_weight_attrs(weight, extra_weight_attrs)
if self.is_checkpoint_serialized:
weight_scale = Parameter(
torch.empty(
@@ -43,6 +43,7 @@ from sglang.multimodal_gen.runtime.layers.linear import (
LinearMethodBase,
ReplicatedLinear,
RowParallelLinear,
UnquantizedLinearMethod,
)
from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import (
ConvRotInt8Config,
@@ -202,20 +203,17 @@ class ConvRotInt8JitLinearMethod(LinearMethodBase):
f"convrot_int8 backend jit needs the output size per "
f"partition ({sum(output_partition_sizes)}) to be a multiple of 8"
)
# The online path initially matches UnquantizedLinearMethod so the
# source weights load in BF16 before quantization. Serialized weights
# allocate their final INT8 storage immediately.
weight = Parameter(
torch.empty(
sum(output_partition_sizes),
input_size_per_partition,
dtype=(torch.int8 if self.is_checkpoint_serialized else params_dtype),
),
requires_grad=False,
# Online checkpoints load their original dtype; serialized ones use INT8.
UnquantizedLinearMethod.create_weights(
self,
layer,
input_size_per_partition,
output_partition_sizes,
input_size,
output_size,
torch.int8 if self.is_checkpoint_serialized else params_dtype,
**extra_weight_attrs,
)
set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0})
layer.register_parameter("weight", weight)
set_weight_attrs(weight, extra_weight_attrs)
if self.is_checkpoint_serialized:
# Comfy stores the per-row scale as [N, 1]; the ops read N contiguous
# floats and accept that shape unchanged.
@@ -21,6 +21,9 @@ from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config impor
from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import (
ConvRotInt8Config,
)
from sglang.multimodal_gen.runtime.layers.quantization.modelopt_fp8 import (
ModelOptFp8LinearMethod as StaticModelOptFp8LinearMethod,
)
from sglang.multimodal_gen.runtime.models.parameter import (
ModelWeightParameter,
PerTensorScaleParameter,
@@ -28,7 +31,6 @@ from sglang.multimodal_gen.runtime.models.parameter import (
from sglang.multimodal_gen.runtime.platforms import current_platform
from sglang.multimodal_gen.runtime.utils.weight_attrs import set_weight_attrs
from sglang.srt.layers.quantization.fp8_utils import (
apply_fp8_linear,
cutlass_fp8_supported,
normalize_e4m3fn_to_e4m3fnuz,
)
@@ -578,20 +580,7 @@ class ModelOptFp8LinearMethod(LinearMethodBase):
copy_or_rebind_param(layer, "weight_scale", processed_weight_scale)
copy_or_rebind_param(layer, "input_scale", layer.input_scale.max())
def apply(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: Optional[torch.Tensor] = None,
) -> torch.Tensor:
return apply_fp8_linear(
input=x,
weight=layer.weight,
weight_scale=layer.weight_scale,
input_scale=layer.input_scale,
bias=bias,
cutlass_fp8_supported=self.cutlass_fp8_supported,
)
apply = StaticModelOptFp8LinearMethod.apply
class ModelOptFp4LinearMethod(LinearMethodBase):
@@ -820,33 +809,9 @@ class ModelOptFp4LinearMethod(LinearMethodBase):
)
x_fp4, x_scale_interleaved = fp4_quantize(x, layer.input_scale_inv)
weights_padding_cols = getattr(layer, "weights_padding_cols", 0)
x_fp4 = pad_nvfp4_activation_for_cutlass(x_fp4, weights_padding_cols)
w = layer.weight
w_scale_interleaved = layer.weight_scale_interleaved
if x_scale_interleaved.dtype == torch.uint8:
x_scale_interleaved = x_scale_interleaved.view(torch.float8_e4m3fn)
if w_scale_interleaved.dtype == torch.uint8:
w_scale_interleaved = w_scale_interleaved.view(torch.float8_e4m3fn)
fp4_gemm, flashinfer_backend = _get_fp4_gemm_op()
if fp4_gemm is None:
raise RuntimeError("No FP4 GEMM kernel available. Install flashinfer.")
out = fp4_gemm(
x_fp4,
w.T,
x_scale_interleaved,
w_scale_interleaved.T,
layer.alpha,
output_dtype,
backend=flashinfer_backend,
out = apply_nvfp4_gemm_prequantized(
layer, x_fp4, x_scale_interleaved, output_dtype, bias
)
out = slice_nvfp4_output(out, output_size)
if bias is not None:
out = out + bias
return out.view(*output_shape)
@@ -44,9 +44,6 @@ logger = init_logger(__name__)
class NPUMXFP4Config(QuantizationConfig):
"""Config for online MXFP4 quantization on NPU (Diffusion)."""
def __init__(self) -> None:
super().__init__()
@classmethod
def get_name(cls) -> str:
return "mxfp4_npu"
@@ -10,7 +10,6 @@ from .mrope import NDRotaryEmbedding, _to_tuple
_ROPE_DICT: dict[tuple, RotaryEmbedding] = {}
_ND_ROPE_CACHE: "OrderedDict[tuple, NDRotaryEmbedding]" = OrderedDict()
_ROPE_3D_CACHE: "OrderedDict[tuple, tuple[torch.Tensor, torch.Tensor]]" = OrderedDict()
def get_rope(
@@ -367,6 +367,26 @@ class NDRotaryEmbedding(torch.nn.Module):
positions = torch.tensor(pos_tuple, dtype=torch.long, device=device)
return self.forward_uncached(pos=positions)
def forward_3d_sequence_shard(
self,
local_len: int,
rank: int,
frame_stride_local: int,
width_local: int,
device: torch.device,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Embed a contiguous shard of a flattened (time, height, width) grid."""
token_start = rank * local_len
token_indices = torch.arange(
token_start, token_start + local_len, device=device, dtype=torch.long
)
t_idx = token_indices // frame_stride_local
rem = token_indices % frame_stride_local
h_idx = rem // width_local
w_idx = rem % width_local
positions = torch.stack((t_idx, h_idx, w_idx), dim=1)
return self.forward_uncached(positions)
def forward_uncached(self, pos: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
"""
The core implementation that computes embeddings from a position tensor.
@@ -6,7 +6,6 @@ from typing import TYPE_CHECKING
import torch
import torch.distributed as dist
import torch.distributed._functional_collectives as ft_c
from torch.distributed.tensor.experimental._attention import _cp_options
from sglang.kernels.ops.diffusion import pack_qkv_destination_major, usp_merge_heads
@@ -28,16 +27,6 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
def _maybe_wait(tensor: torch.Tensor) -> torch.Tensor:
"""
When tracing the code, the result tensor is not an AsyncCollectiveTensor,
so we cannot call ``wait()``.
"""
if isinstance(tensor, ft_c.AsyncCollectiveTensor):
return tensor.wait()
return tensor
_A2A_STAGING_BUFFERS: dict[tuple[str, torch.dtype, int], torch.Tensor] = {}
@@ -79,13 +79,7 @@ def direct_register_custom_op(
# Operator doesn't exist, proceed with registration
pass
if hasattr(torch.library, "infer_schema"):
schema_str = torch.library.infer_schema(op_func, mutates_args=mutates_args)
else:
# for pytorch 2.4
import torch._custom_op.impl
schema_str = torch._custom_op.impl.infer_schema(op_func, mutates_args)
schema_str = torch.library.infer_schema(op_func, mutates_args=mutates_args)
try:
my_lib.define(op_name + schema_str)
@@ -106,9 +100,6 @@ def direct_register_custom_op(
else:
# Re-raise other RuntimeErrors
raise error
except AttributeError as error:
# Always re-raise AttributeError as it indicates missing dependencies
raise error
class CustomOpWrapper:
@@ -15,6 +15,7 @@ from sglang.multimodal_gen.runtime.distributed import (
get_tp_group,
tensor_model_parallel_all_reduce,
)
from sglang.multimodal_gen.runtime.layers.linear import UnquantizedLinearMethod
from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config import (
QuantizationConfig,
QuantizeMethodBase,
@@ -23,7 +24,6 @@ from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config impor
from sglang.multimodal_gen.runtime.layers.utils import get_group_rank, get_group_size
from sglang.multimodal_gen.runtime.models.parameter import BasevLLMParameter
from sglang.multimodal_gen.runtime.platforms import current_platform
from sglang.multimodal_gen.runtime.utils.weight_attrs import set_weight_attrs
DEFAULT_VOCAB_PADDING_SIZE = 64
@@ -31,29 +31,7 @@ DEFAULT_VOCAB_PADDING_SIZE = 64
class UnquantizedEmbeddingMethod(QuantizeMethodBase):
"""Unquantized method for embeddings."""
def create_weights(
self,
layer: torch.nn.Module,
input_size_per_partition: int,
output_partition_sizes: list[int],
input_size: int,
output_size: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
):
"""Create weights for embedding layer."""
weight = Parameter(
torch.empty(
sum(output_partition_sizes),
input_size_per_partition,
dtype=params_dtype,
),
requires_grad=False,
)
set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0})
layer.register_parameter("weight", weight)
set_weight_attrs(weight, extra_weight_attrs)
create_weights = UnquantizedLinearMethod.create_weights
def apply(
self, layer: torch.nn.Module, x: torch.Tensor, bias: torch.Tensor | None = None
@@ -344,71 +322,6 @@ class VocabParallelEmbedding(torch.nn.Module):
added_vocab_end_index,
)
def get_sharded_to_full_mapping(self) -> list[int] | None:
"""Get a mapping that can be used to reindex the gathered
logits for sampling.
During sampling, we gather logits from all ranks. The relationship
of index->token_id will follow the same format as outlined in the class
docstring. However, after the gather, we want to reindex the final
logits tensor to map index->token_id one-to-one (the index is always
equal the token_id it corresponds to). The indices returned by this
method allow us to do that.
"""
if self.tp_size < 2:
return None
base_embeddings: list[int] = []
added_embeddings: list[int] = []
padding: list[int] = []
for tp_rank in range(self.tp_size):
shard_indices = self._get_indices(
self.num_embeddings_padded,
self.org_vocab_size_padded,
self.num_embeddings,
self.org_vocab_size,
tp_rank,
self.tp_size,
)
range_start = self.num_embeddings_per_partition * tp_rank
range_end = self.num_embeddings_per_partition * (tp_rank + 1)
base_embeddings.extend(
range(range_start, range_start + shard_indices.num_org_elements)
)
padding.extend(
range(
range_start + shard_indices.num_org_elements,
range_start + shard_indices.num_org_elements_padded,
)
)
added_embeddings.extend(
range(
range_start + shard_indices.num_org_elements_padded,
range_start
+ shard_indices.num_org_elements_padded
+ shard_indices.num_added_elements,
)
)
padding.extend(
range(
range_start
+ shard_indices.num_org_elements_padded
+ shard_indices.num_added_elements,
range_start
+ shard_indices.num_org_elements_padded
+ shard_indices.num_added_elements_padded,
)
)
assert (
range_start
+ shard_indices.num_org_elements_padded
+ shard_indices.num_added_elements_padded
== range_end
)
ret = base_embeddings + added_embeddings + padding
assert len(ret) == self.num_embeddings_padded
return ret
def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor):
output_dim = getattr(param, "output_dim", None)
packed_dim = getattr(param, "packed_dim", None)
@@ -5,7 +5,7 @@ from collections.abc import Generator
import torch
import transformers
from torch import nn
from transformers import PretrainedConfig
from transformers import AutoConfig, PretrainedConfig
from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
from sglang.multimodal_gen.configs.models import EncoderConfig
@@ -83,7 +83,6 @@ from sglang.multimodal_gen.runtime.models.registry import ModelRegistry
from sglang.multimodal_gen.runtime.platforms import current_platform
from sglang.multimodal_gen.runtime.server_args import ServerArgs
from sglang.multimodal_gen.runtime.utils.hf_diffusers_utils import (
get_config,
get_diffusers_component_config,
load_dict,
)
@@ -568,7 +567,7 @@ class TextEncoderLoader(OnlineQuantizationComponentLoader):
server_args: ServerArgs,
component_name: str,
) -> EncoderConfig:
diffusers_pretrained_config = get_config(
diffusers_pretrained_config = AutoConfig.from_pretrained(
component_model_path, trust_remote_code=True
)
encoder_index = self._extract_encoder_index(
@@ -118,8 +118,6 @@ def _convert_conv3d_weights_to_channels_last_3d(module: nn.Module) -> int:
Convert Conv3d weights to channels_last_3d (NDHWC) memory format.
Returns the number of Conv3d modules converted.
"""
if not hasattr(torch, "channels_last_3d"):
return 0
num_converted = 0
for m in module.modules():
if isinstance(m, nn.Conv3d):
@@ -5,7 +5,6 @@
"""Utilities for downloading, loading, initializing and verifying model weights."""
import hashlib
import json
import os
import tempfile
from collections import defaultdict
@@ -139,38 +138,6 @@ def get_lock(model_name_or_path: str | Path, cache_dir: str | None = None):
return lock
# For models like Mistral-7B-v0.3, there are both sharded
# safetensors files and a consolidated safetensors file.
# Passing both of these to the weight loader functionality breaks.
# So, we use the index_file to
# look up which safetensors files should be used.
def filter_duplicate_safetensors_files(
hf_weights_files: list[str],
hf_folder: str,
index_file: str,
key_filter: Callable[[str], bool] | None = None,
) -> list[str]:
# model.safetensors.index.json is a mapping from keys in the
# torch state_dict to safetensors file holding that weight.
index_file_name = os.path.join(hf_folder, index_file)
if not os.path.isfile(index_file_name):
return hf_weights_files
# Iterate through the weight_map (weight_name: safetensors files)
# to identify weights that we should use.
with open(index_file_name) as f:
weight_map = json.load(f)["weight_map"]
weight_files_in_index = set()
for weight_name in weight_map:
# remove only shards whose indexed tensors are all filtered
if key_filter is not None and not key_filter(weight_name):
continue
weight_files_in_index.add(os.path.join(hf_folder, weight_map[weight_name]))
# Filter out any fields that are not found in the index file.
hf_weights_files = [f for f in hf_weights_files if f in weight_files_in_index]
return hf_weights_files
def filter_files_not_needed_for_inference(hf_weights_files: list[str]) -> list[str]:
"""
Exclude files that are not needed for inference.
@@ -376,23 +343,18 @@ def pt_weights_iterator(
def default_weight_loader(param: torch.Tensor, loaded_weight: torch.Tensor) -> None:
"""Default weight loader."""
try:
if param.numel() == 1 and loaded_weight.numel() == 1:
# Sometimes scalar values aren't considered tensors with shapes
# so if both param and loaded_weight are a scalar,
# "broadcast" instead of copy
param.data.fill_(loaded_weight.item())
else:
assert param.size() == loaded_weight.size(), (
f"Attempted to load weight ({loaded_weight.size()}) "
f"into parameter ({param.size()})"
)
if param.numel() == 1 and loaded_weight.numel() == 1:
# Sometimes scalar values aren't considered tensors with shapes
# so if both param and loaded_weight are a scalar,
# "broadcast" instead of copy
param.data.fill_(loaded_weight.item())
else:
assert param.size() == loaded_weight.size(), (
f"Attempted to load weight ({loaded_weight.size()}) "
f"into parameter ({param.size()})"
)
param.data.copy_(loaded_weight)
except Exception:
# NOTE: This exception is added for the purpose of setting breakpoint to
# debug weight loading issues.
raise
param.data.copy_(loaded_weight)
def load_stacked_weight(
@@ -485,6 +447,36 @@ def maybe_remap_kv_scale_name(name: str, params_dict: dict) -> str | None:
return name
def load_llm_encoder_weights(
weights: Iterable[tuple[str, torch.Tensor]],
params_dict: dict[str, torch.nn.Parameter],
stacked_params_mapping: Iterable[tuple[str, str, str | int]],
*,
strip_prefix: str = "",
) -> set[str]:
"""Load Llama/Qwen-style encoder weights, excluding rotary cache tensors."""
loaded_params: set[str] = set()
for name, loaded_weight in weights:
if strip_prefix and name.startswith(strip_prefix):
name = name[len(strip_prefix) :]
if (
"rotary_emb.inv_freq" in name
or "rotary_emb.cos_cached" in name
or "rotary_emb.sin_cached" in name
):
continue
if "scale" in name:
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None:
continue
name = load_stacked_weight(
name, loaded_weight, params_dict, stacked_params_mapping
)
if name is not None:
loaded_params.add(name)
return loaded_params
def compute_weights_checksum(
named_params: Iterable[tuple[str, torch.Tensor]],
) -> str:
@@ -2,51 +2,20 @@
# SPDX-License-Identifier: Apache-2.0
# Adapted from vllm: https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/forward_context.py
import time
from collections import defaultdict
from contextlib import contextmanager
from dataclasses import dataclass
from typing import TYPE_CHECKING, Optional, Type
import torch
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
from typing import TYPE_CHECKING, Optional
if TYPE_CHECKING:
from sglang.multimodal_gen.runtime.layers.attention import AttentionMetadata
from sglang.multimodal_gen.runtime.pipelines_core import Req
logger = init_logger(__name__)
# TODO(will): check if this is needed
# track_batchsize: bool = envs.SGLANG_DIFFUSION_LOG_BATCHSIZE_INTERVAL >= 0
track_batchsize: bool = False
last_logging_time: float = 0
forward_start_time: float = 0
# batchsize_logging_interval: float = envs.SGLANG_DIFFUSION_LOG_BATCHSIZE_INTERVAL
batchsize_logging_interval: float = 1000
batchsize_forward_time: defaultdict = defaultdict(list)
@dataclass
class ForwardContext:
current_timestep: int
# TODO(will): check this arg
# copy from vllm_config.compilation_config.static_forward_context
# attn_layers: Dict[str, Any]
# TODO: extend to support per-layer dynamic forward context
attn_metadata: "AttentionMetadata" # set dynamically for each forward pass
forward_batch: Optional["Req"] = None
attention_backend_cls: Optional[Type] = None
def set_attn_backend_cls(self, attention_backend_cls: Type):
if self.attention_backend_cls:
if self.attention_backend_cls != attention_backend_cls:
raise RuntimeError(
f"Different types of attention backend in a same context detected, previous: {self.attention_backend_cls}, new: {attention_backend_cls}"
)
else:
self.attention_backend_cls = attention_backend_cls
_forward_context: Optional["ForwardContext"] = None
@@ -65,7 +34,6 @@ def get_forward_context_or_none() -> "ForwardContext | None":
return _forward_context
# TODO(will): finalize the interface
@contextmanager
def set_forward_context(
current_timestep, attn_metadata, forward_batch: Optional["Req"] = None
@@ -74,10 +42,6 @@ def set_forward_context(
can be attention metadata, etc.
Here we can inject common logic for every model forward pass.
"""
global forward_start_time
need_to_track_batchsize = track_batchsize and attn_metadata is not None
if need_to_track_batchsize:
forward_start_time = time.perf_counter()
global _forward_context
prev_context = _forward_context
_forward_context = ForwardContext(
@@ -89,36 +53,4 @@ def set_forward_context(
try:
yield
finally:
global last_logging_time, batchsize_logging_interval
if need_to_track_batchsize:
if hasattr(attn_metadata, "num_prefill_tokens"):
# for v0 attention backends
batchsize = (
attn_metadata.num_prefill_tokens + attn_metadata.num_decode_tokens
)
else:
# for v1 attention backends
batchsize = attn_metadata.num_input_tokens
now = time.perf_counter()
# time measurement is in milliseconds
batchsize_forward_time[batchsize].append((now - forward_start_time) * 1000)
if now - last_logging_time > batchsize_logging_interval:
last_logging_time = now
forward_stats = []
for bs, times in batchsize_forward_time.items():
if len(times) <= 1:
# can be cudagraph / profiling run
continue
medium = torch.quantile(torch.tensor(times), q=0.5).item()
medium = round(medium, 2)
forward_stats.append((bs, len(times), medium))
forward_stats.sort(key=lambda x: x[1], reverse=True)
if forward_stats:
logger.info(
(
"Batchsize forward time stats "
"(batchsize, count, median_time(ms)): %s"
),
forward_stats,
)
_forward_context = prev_context
@@ -1,25 +1,9 @@
# SPDX-License-Identifier: Apache-2.0
"""Warmup-calibrated automatic component residency placement.
"""Warmup memory, timing and layer-use estimates for the default workload.
Under ``--performance-mode auto`` with server warmup, each rank measures the
peak GPU memory of bounded synthetic warmup requests and a low-step probe at
the complete default serving shape, then selects a complete serving placement
for every eligible component under the measured memory constraints.
When no full-shape measurement is available, the fallback estimate splits the
measured peak into persistent weights and workload-scaled activations. Scaling
the whole peak would multiply resident weights by the video frame/area cap
ratio (~16x for Wan-class defaults) and residency adjustment would never trigger.
The planner targets the model default workload only (default resolution,
default frames, batch=1). Larger shapes, batches, or multi-image inputs need
explicit ``--component-residency``.
Loading and serving are deliberately separate placement states. The existing
auto policy provides the initial state; when that state can complete loading
and calibration, this module optimizes the long-lived serving state and
validates the transition with a post-placement warmup. It does not force a
single placement to serve two different lifecycle objectives.
Peak-memory extrapolation separates persistent weights from workload-scaled
activations. Phase and layer-use estimates account for stage-specific iteration
counts rather than scaling every stage by the request's denoising steps.
"""
from __future__ import annotations
@@ -29,61 +13,15 @@ from typing import Iterable, Mapping
import msgspec
from sglang.multimodal_gen.runtime.managers.memory_managers.component_residency import (
COMPONENT_OFFLOAD,
LAYERWISE_OFFLOAD,
RESIDENT,
)
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload_components import (
is_dit_component_name,
)
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
logger = init_logger(__name__)
GIB_BYTES = 1024**3
# Activation memory rarely scales perfectly linearly with workload units;
# pad the extrapolated activation part before checking the budget.
ACTIVATION_EXTRAPOLATION_MARGIN = 1.2
# A target-shape measurement plus the mandatory post-placement warmup justifies
# a tighter reserve than an extrapolated estimate. Both retain an absolute
# floor for allocator slack, shape variance, and CUDA graph or compile pools.
MEASURED_VRAM_RESERVE_FRACTION = 0.05
EXTRAPOLATED_VRAM_RESERVE_FRACTION = 0.10
MIN_VRAM_RESERVE_BYTES = 4 * GIB_BYTES
# The absolute floor is sized for datacenter cards, where either fraction can
# dominate. On a 12 GiB card a flat 4 GiB would fence off a third of the device,
# so cap the floor as a share of what is actually there.
MAX_VRAM_RESERVE_FRACTION = 0.20
# A feasible placement is not automatically useful. Predictions inside this
# interval are treated as latency-equivalent. The joint optimizer then avoids
# changing strategy, minimizes device memory, preserves the faster estimate,
# and finally minimizes HostPin. The raw estimate is already an upper bound:
# transfer time is capped by the measured request.
ESTIMATED_PINNED_H2D_BYTES_PER_SECOND = 24 * GIB_BYTES
MIN_LATENCY_EQUIVALENCE_NS = 50_000_000
MAX_LATENCY_EQUIVALENCE_NS = 100_000_000
LATENCY_EQUIVALENCE_FRACTION = 0.01
# The transfer model ranks feasible placements; the mandatory warmup is the
# authority on whether a selected placement actually helped. Allow normal
# measurement noise, but undo a round whose calibrated request is materially
# slower than the original layout.
POST_ADJUSTMENT_REGRESSION_FRACTION = 0.05
PLACEMENT_STATUS_SKIPPED = "skipped"
PLACEMENT_STATUS_ADJUSTED = "adjusted"
PLACEMENT_STATUS_VALIDATED = "validated"
PLACEMENT_STATUS_ROLLED_BACK = "rolled_back"
PLACEMENT_STATUS_ROLLBACK_FAILED = "rollback_failed"
def describe_error(error: BaseException) -> str:
"""Never-empty error text (str(AssertionError()) is "" and would be
dropped by any truthiness filter)."""
text = str(error)
return f"{type(error).__name__}: {text}" if text else type(error).__name__
class WarmupMemoryRecord(msgspec.Struct, frozen=True):
@@ -113,96 +51,6 @@ class WarmupMemoryRecord(msgspec.Struct, frozen=True):
return max(1, self.width) * max(1, self.height) * max(1, self.num_frames)
class ResidencyTarget(msgspec.Struct, frozen=True):
"""One complete target state for an auto-managed component."""
component_name: str
residency_mode: str
target_resident_weight_bytes: int
# Estimated per-request host-to-device traffic this target removes.
h2d_bytes_per_request: int
# Layerwise candidates jointly choose stage-scoped GPU residency and host
# pinning. None is used by ordinary component placement.
target_layerwise_resident_layers: tuple[int, ...] | None = None
target_layerwise_pinned_layers: tuple[tuple[int, ...], ...] | None = None
pinned_host_delta_bytes: int = 0
host_unpin_scratch_bytes: int = 0
host_pin_scratch_bytes: int = 0
host_materialize_scratch_bytes: int = 0
# Signed device-memory delta while applying the placement before the
# validation warmup. Layerwise -> resident materializes every managed
# layer immediately; a demotion can release those bytes first and fund a
# later materialization in the same transaction.
device_transition_delta_bytes: int = 0
permanent_residency: bool = False
# Device-memory delta relative to the measured placement. A component
# already loaded for its own phase has a different delta from phases where
# it is absent; keeping both avoids adding the same weights twice.
active_device_delta_bytes: int = 0
# Delta when the component is already present because of async prefetch,
# but is not the semantic owner of this phase.
present_device_delta_bytes: int = 0
inactive_device_delta_bytes: int = 0
# None preserves the historical derived target for hand-built callers:
# partial layerwise targets remain layerwise, every other option is
# resident. Generated complete-state frontiers set this explicitly.
target_residency_mode: str | None = None
current_placement: bool = False
target_device_weight_bytes: int = 0
target_pinned_host_bytes: int = 0
def target_mode(self) -> str:
if self.target_residency_mode is not None:
return self.target_residency_mode
if (
self.target_layerwise_resident_layers is not None
and not self.permanent_residency
):
return LAYERWISE_OFFLOAD
return RESIDENT
def option_key(self) -> str:
target_mode = self.target_mode()
if target_mode == COMPONENT_OFFLOAD:
return f"{self.component_name}:component-offload"
if self.target_layerwise_resident_layers is None:
return f"{self.component_name}:resident"
layer_counts = ",".join(
str(count) for count in self.target_layerwise_resident_layers
)
pinned = "|".join(
",".join(str(index) for index in indices) or "-"
for indices in self.target_layerwise_pinned_layers or ()
)
permanence = "permanent" if self.permanent_residency else "stage"
return f"{self.component_name}:{permanence}:layers={layer_counts}:pins={pinned}"
class RankResidencyReport(msgspec.Struct, frozen=True):
"""One rank's inputs to the replica-wide placement decision."""
rank: int
budget_bytes: int
estimated_peak_bytes: int | None
target_workload_measured: bool = False
observed_reserved_bytes: int = 0
estimated_peak_bytes_by_phase: dict[str, int] = {}
active_components_by_phase: dict[str, tuple[str, ...]] = {}
used_components_by_phase: dict[str, tuple[str, ...]] = {}
full_weight_transition_components_by_phase: dict[str, tuple[str, ...]] = {}
current_device_weight_bytes_by_component: dict[str, int] = {}
node_rank: int = 0
pinned_host_bytes: int = 0
host_pin_capacity_bytes: int = 0
host_transition_headroom_bytes: int = 0
device_transition_allocated_bytes: int = 0
estimated_request_duration_ns: int = 0
measured_request_duration_ns: int = 0
candidate_latency_savings_ns: dict[str, int] = {}
candidates: list[ResidencyTarget] = []
skip_reason: str | None = None
def estimate_layerwise_layer_uses(
*,
records: Iterable[WarmupMemoryRecord],
@@ -12,6 +12,7 @@ import os
from dataclasses import dataclass
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload_components import (
component_base_name,
is_dit_component_name,
is_image_encoder_component_name,
is_text_encoder_component_name,
@@ -34,13 +35,6 @@ class ComponentLoadSpec:
_WEIGHT_FILE_SUFFIXES = (".bin", ".pt", ".pth")
def _component_base_name(component_name: str) -> str:
prefix, separator, suffix = component_name.rpartition("_")
if separator and suffix.isdigit():
return prefix
return component_name
def _component_variant_priority(component_name: str) -> int:
_, separator, suffix = component_name.rpartition("_")
if separator and suffix.isdigit():
@@ -50,7 +44,7 @@ def _component_variant_priority(component_name: str) -> int:
def component_load_risk_rank(component_name: str) -> int:
"""Fallback type rank when checkpoint size cannot be inferred."""
candidate_names = (component_name, _component_base_name(component_name))
candidate_names = (component_name, component_base_name(component_name))
if any(is_dit_component_name(name) for name in candidate_names):
return 0
if any(is_text_encoder_component_name(name) for name in candidate_names):
@@ -98,7 +98,6 @@ class CausalWanSelfAttention(nn.Module):
self.sink_size = sink_size
self.qk_norm = qk_norm
self.eps = eps
self.parallel_attention = parallel_attention
# Scaled dot product attention
self.attn = LocalAttention(
@@ -375,7 +375,6 @@ class DomainAwareLinear(nn.Module):
super().__init__()
self.input_size = input_size
self.output_size = output_size
self.num_domains = num_domains
self.fc = nn.Embedding(num_domains, output_size * input_size)
self.bias = nn.Embedding(num_domains, output_size)
nn.init.xavier_uniform_(
@@ -583,6 +582,27 @@ def _build_mlp(
# -----------------------------------------------------------------------------
def _initialize_attention_projections(module, prefix, quant_config):
module.q_size = module.num_attention_heads * module.head_dim
module.kv_size = module.num_key_value_heads * module.head_dim
module.to_qkv = MergedColumnParallelLinear(
module.hidden_size,
[module.q_size, module.kv_size, module.kv_size],
bias=False,
gather_output=False,
quant_config=quant_config,
prefix=add_prefix("to_qkv", prefix),
)
module.to_out = RowParallelLinear(
module.q_size,
module.hidden_size,
bias=False,
input_is_parallel=True,
quant_config=quant_config,
prefix=add_prefix("to_out", prefix),
)
class Cosmos3CausalAttention(nn.Module):
"""Understanding pathway: causal self-attention on text tokens."""
@@ -618,24 +638,7 @@ class Cosmos3CausalAttention(nn.Module):
self.local_num_attention_heads = num_attention_heads // self.tp_size
self.local_num_key_value_heads = num_key_value_heads // self.tp_size
self.q_size = num_attention_heads * head_dim
self.kv_size = num_key_value_heads * head_dim
self.to_qkv = MergedColumnParallelLinear(
hidden_size,
[self.q_size, self.kv_size, self.kv_size],
bias=False,
gather_output=False,
quant_config=quant_config,
prefix=add_prefix("to_qkv", prefix),
)
self.to_out = RowParallelLinear(
num_attention_heads * head_dim,
hidden_size,
bias=False,
input_is_parallel=True,
quant_config=quant_config,
prefix=add_prefix("to_out", prefix),
)
_initialize_attention_projections(self, prefix, quant_config)
# Per-head QK norm (optional; some backbones omit it on text).
if qk_norm:
@@ -754,24 +757,7 @@ class Cosmos3CrossAttention(nn.Module):
self.local_num_attention_heads = num_attention_heads // self.tp_size
self.local_num_key_value_heads = num_key_value_heads // self.tp_size
self.q_size = num_attention_heads * head_dim
self.kv_size = num_key_value_heads * head_dim
self.to_qkv = MergedColumnParallelLinear(
hidden_size,
[self.q_size, self.kv_size, self.kv_size],
bias=False,
gather_output=False,
quant_config=quant_config,
prefix=add_prefix("to_qkv", prefix),
)
self.to_out = RowParallelLinear(
num_attention_heads * head_dim,
hidden_size,
bias=False,
input_is_parallel=True,
quant_config=quant_config,
prefix=add_prefix("to_out", prefix),
)
_initialize_attention_projections(self, prefix, quant_config)
self.norm_q = RMSNorm(head_dim, eps=1e-6)
self.norm_k = RMSNorm(head_dim, eps=1e-6)
@@ -523,7 +523,6 @@ class FluxAttention(torch.nn.Module, AttentionModuleMixin):
)
self.local_heads = divide(self.heads, self.tp_size)
self.added_kv_proj_dim = added_kv_proj_dim
self.added_proj_bias = added_proj_bias
self.use_fused_qkv = isinstance(quant_config, NunchakuConfig)
self.use_fused_added_qkv = isinstance(quant_config, NunchakuConfig)
@@ -573,7 +573,6 @@ class Flux2Attention(torch.nn.Module, AttentionModuleMixin):
self.dropout = dropout
self.added_kv_proj_dim = added_kv_proj_dim
self.added_proj_bias = added_proj_bias
# Packed NVFP4 checkpoints already serialize QKV together. ModelOpt
# FP8 exports separate Diffusers tensors, but the loader can merge
@@ -669,7 +669,6 @@ class HunyuanVideoTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixi
self.out_channels = (
config.in_channels if config.out_channels is None else config.out_channels
)
self.unpatchify_channels = self.out_channels
self.guidance_embeds = config.guidance_embeds
self.rope_dim_list = list(config.rope_axes_dim)
self.rope_theta = config.rope_theta
@@ -704,16 +704,9 @@ class LingBotWorldTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixi
width_local: int,
device: torch.device,
) -> tuple[torch.Tensor, torch.Tensor]:
token_start = rank * local_len
token_indices = torch.arange(
token_start, token_start + local_len, device=device, dtype=torch.long
return self.rotary_emb.forward_3d_sequence_shard(
local_len, rank, frame_stride_local, width_local, device
)
t_idx = token_indices // frame_stride_local
rem = token_indices % frame_stride_local
h_idx = rem // width_local
w_idx = rem % width_local
positions = torch.stack((t_idx, h_idx, w_idx), dim=1)
return self.rotary_emb.forward_uncached(positions)
def forward(
self,
@@ -520,11 +520,6 @@ class LTX2AudioVideoRotaryPosEmbed(nn.Module):
audio_coords = audio_coords.unsqueeze(1)
return audio_coords
def prepare_coords(self, *args, **kwargs):
if self.modality == "video":
return self.prepare_video_coords(*args, **kwargs)
return self.prepare_audio_coords(*args, **kwargs)
def forward(
self,
coords: torch.Tensor,
@@ -697,7 +692,6 @@ class LTX2TPRMSNormAcrossHeads(nn.Module):
) -> None:
super().__init__()
self.full_hidden_size = full_hidden_size
self.local_hidden_size = local_hidden_size
self.eps = eps
self.weight = nn.Parameter(torch.ones(local_hidden_size))
@@ -1864,8 +1858,6 @@ class LTX2VideoTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixin):
num_attention_heads=self.audio_num_attention_heads,
)
self.cross_pe_max_pos = cross_attn_pos_embed_max_pos
# 5. Transformer Blocks
self.transformer_blocks = nn.ModuleList(
[
@@ -15,7 +15,6 @@ from torch.distributed.tensor import DTensor
from sglang.multimodal_gen.configs.models.dits.mova_audio import MOVAAudioConfig
from sglang.multimodal_gen.configs.models.fsdp import is_block
from sglang.multimodal_gen.runtime.layers.linear import ReplicatedLinear
from sglang.multimodal_gen.runtime.layers.mlp import MLP
from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config import (
QuantizationConfig,
)
@@ -25,7 +24,11 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload im
from sglang.multimodal_gen.runtime.models.dits.base import CachableDiT
# Reuse common functions and classes from mova_video_dit
from .mova_video_dit import DiTBlock, precompute_freqs_cis, sinusoidal_embedding_1d
from .mova_video_dit import (
initialize_mova_transformer_layers,
precompute_freqs_cis,
sinusoidal_embedding_1d,
)
def precompute_freqs_cis_1d(dim: int, end: int = 16384, theta: float = 10000.0):
@@ -102,15 +105,11 @@ class WanAudioModel(CachableDiT, LayerwiseOffloadableModuleMixin):
# Extract parameters from config
dim = config.dim
in_dim = config.in_dim
ffn_dim = config.ffn_dim
out_dim = config.out_dim
text_dim = config.text_dim
freq_dim = config.freq_dim
eps = config.eps
patch_size = config.patch_size
num_heads = config.num_heads
num_layers = config.num_layers
has_image_pos_emb = config.has_image_pos_emb
has_ref_conv = config.has_ref_conv
separated_timestep = config.separated_timestep
require_vae_embedding = config.require_vae_embedding
@@ -126,38 +125,15 @@ class WanAudioModel(CachableDiT, LayerwiseOffloadableModuleMixin):
self.require_clip_embedding = require_clip_embedding
self.fuse_vae_embedding_in_latents = fuse_vae_embedding_in_latents
self.vae_type = vae_type
# self.patch_embedding = nn.Conv3d(
# in_dim, dim, kernel_size=patch_size, stride=patch_size)
self.patch_embedding = Conv1dLocalIsland(
in_dim, dim, kernel_size=patch_size, stride=patch_size
)
self.text_embedding = MLP(
text_dim,
dim,
output_dim=dim,
act_type="gelu_pytorch_tanh",
quant_config=quant_config,
)
self.time_embedding = MLP(
freq_dim, dim, output_dim=dim, act_type="silu", quant_config=quant_config
)
# Preserve state_dict keys (time_projection.1.weight/bias).
self.time_projection = nn.Sequential(
nn.SiLU(), ReplicatedLinear(dim, dim * 6, quant_config=quant_config)
)
self.blocks = nn.ModuleList(
[
DiTBlock(dim, num_heads, ffn_dim, eps, quant_config=quant_config)
for _ in range(num_layers)
]
)
initialize_mova_transformer_layers(self, config, quant_config)
self.head = Head(dim, out_dim, patch_size, eps)
self.num_heads = num_heads
self.freqs = None
self.img_pos_emb = None
if has_ref_conv:
self.ref_conv = nn.Conv2d(16, dim, kernel_size=(2, 2), stride=(2, 2))
self.has_image_pos_emb = has_image_pos_emb
self.has_ref_conv = has_ref_conv
self.hidden_size = dim
self.num_attention_heads = num_heads
@@ -272,9 +272,6 @@ class CrossAttention(_MOVAAttention):
class MulAdd(nn.Module):
def __init__(self):
super().__init__()
def forward(self, x, gate, residual):
return residual + gate * x
@@ -408,6 +405,41 @@ class Conv3dLocalIsland(nn.Conv3d):
return super().forward(input)
def initialize_mova_transformer_layers(model, config, quant_config):
"""Create shared audio/video layers in checkpoint registration order."""
model.text_embedding = MLP(
config.text_dim,
config.dim,
output_dim=config.dim,
act_type="gelu_pytorch_tanh",
quant_config=quant_config,
)
model.time_embedding = MLP(
config.freq_dim,
config.dim,
output_dim=config.dim,
act_type="silu",
quant_config=quant_config,
)
# Preserve state_dict keys (time_projection.1.weight/bias).
model.time_projection = nn.Sequential(
nn.SiLU(),
ReplicatedLinear(config.dim, config.dim * 6, quant_config=quant_config),
)
model.blocks = nn.ModuleList(
[
DiTBlock(
config.dim,
config.num_heads,
config.ffn_dim,
config.eps,
quant_config=quant_config,
)
for _ in range(config.num_layers)
]
)
class WanModel(CachableDiT, LayerwiseOffloadableModuleMixin):
_fsdp_shard_conditions = [is_block]
_compile_conditions = [is_block]
@@ -426,15 +458,11 @@ class WanModel(CachableDiT, LayerwiseOffloadableModuleMixin):
# Extract parameters from config
dim = config.dim
in_dim = config.in_dim
ffn_dim = config.ffn_dim
out_dim = config.out_dim
text_dim = config.text_dim
freq_dim = config.freq_dim
eps = config.eps
patch_size = config.patch_size
num_heads = config.num_heads
num_layers = config.num_layers
has_image_pos_emb = config.has_image_pos_emb
has_ref_conv = config.has_ref_conv
separated_timestep = config.separated_timestep
require_vae_embedding = config.require_vae_embedding
@@ -452,33 +480,13 @@ class WanModel(CachableDiT, LayerwiseOffloadableModuleMixin):
self.patch_embedding = Conv3dLocalIsland(
in_dim, dim, kernel_size=patch_size, stride=patch_size
)
self.text_embedding = MLP(
text_dim,
dim,
output_dim=dim,
act_type="gelu_pytorch_tanh",
quant_config=quant_config,
)
self.time_embedding = MLP(
freq_dim, dim, output_dim=dim, act_type="silu", quant_config=quant_config
)
# Preserve state_dict keys (time_projection.1.weight/bias).
self.time_projection = nn.Sequential(
nn.SiLU(), ReplicatedLinear(dim, dim * 6, quant_config=quant_config)
)
self.blocks = nn.ModuleList(
[
DiTBlock(dim, num_heads, ffn_dim, eps, quant_config=quant_config)
for _ in range(num_layers)
]
)
initialize_mova_transformer_layers(self, config, quant_config)
self.head = Head(dim, out_dim, patch_size, eps)
self.num_heads = num_heads
self.freqs = None
if has_ref_conv:
self.ref_conv = nn.Conv2d(16, dim, kernel_size=(2, 2), stride=(2, 2))
self.has_image_pos_emb = has_image_pos_emb
self.has_ref_conv = has_ref_conv
self.hidden_size = dim
self.num_attention_heads = num_heads
@@ -744,7 +744,6 @@ class QwenImageCrossAttention(nn.Module):
self.window_size = window_size
self.qk_norm = qk_norm
self.eps = eps
self.parallel_attention = parallel_attention
self.added_kv_proj_dim = added_kv_proj_dim
self.prefix = prefix
self.defer_output_bias = _defer_modelopt_output_bias(quant_config)
@@ -169,6 +169,17 @@ class SanaWMBlock(nn.Module):
tokens_per_frame = N // num_frames
return x.reshape(B, num_frames, tokens_per_frame, C), tokens_per_frame
def _prepare_modulation(self, t: torch.Tensor, batch_size: int):
if t.dim() == 2:
return None, (
self.scale_shift_table[None] + t.reshape(batch_size, 6, -1)
).chunk(6, dim=1)
num_frames = t.reshape(batch_size, -1, 6, t.shape[-1] // 6).shape[1]
t = t.reshape(batch_size, num_frames, 6, -1)
return num_frames, (self.scale_shift_table[None, None, :, :] + t).chunk(
6, dim=2
)
def forward(
self,
x: torch.Tensor, # (B, N, D)
@@ -184,17 +195,8 @@ class SanaWMBlock(nn.Module):
chunk_index: Optional[List[int]] = None,
) -> torch.Tensor:
B = x.shape[0]
if t.dim() == 2:
num_frames = None
shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = (
self.scale_shift_table[None] + t.reshape(B, 6, -1)
).chunk(6, dim=1)
else:
num_frames = t.reshape(B, -1, 6, t.shape[-1] // 6).shape[1]
t = t.reshape(B, num_frames, 6, -1)
shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = (
self.scale_shift_table[None, None, :, :] + t
).chunk(6, dim=2)
num_frames, modulation = self._prepare_modulation(t, B)
shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = modulation
# Self-attention with UCPE camera branch
if num_frames is None:
@@ -259,17 +261,8 @@ class SanaWMBlock(nn.Module):
) -> Tuple[torch.Tensor, list]:
"""Streaming counterpart of ``forward``: threads the per-block 10-slot ``kv_cache`` through cached attention + FFN."""
B = x.shape[0]
if t.dim() == 2:
num_frames = None
shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = (
self.scale_shift_table[None] + t.reshape(B, 6, -1)
).chunk(6, dim=1)
else:
num_frames = t.reshape(B, -1, 6, t.shape[-1] // 6).shape[1]
t = t.reshape(B, num_frames, 6, -1)
shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = (
self.scale_shift_table[None, None, :, :] + t
).chunk(6, dim=2)
num_frames, modulation = self._prepare_modulation(t, B)
shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = modulation
if num_frames is None:
x_in = self._modulate(self.norm1(x), shift_msa, scale_msa)
@@ -435,7 +428,6 @@ class SanaWMTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixin):
for i in range(depth)
if arch.softmax_every_n > 0 and (i + 1) % arch.softmax_every_n == 0
)
self.softmax_block_indices = tuple(sorted(softmax_idx))
self.blocks = nn.ModuleList(
[
@@ -203,7 +203,6 @@ class WanSelfAttention(nn.Module):
self.window_size = window_size
self.qk_norm = qk_norm
self.eps = eps
self.parallel_attention = parallel_attention
tp_size = get_tp_world_size()
# layers
@@ -1137,19 +1136,9 @@ class WanTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixin):
width_local: int,
device: torch.device,
) -> tuple[torch.Tensor, torch.Tensor]:
token_start = rank * local_len
token_indices = torch.arange(
token_start,
token_start + local_len,
device=device,
dtype=torch.long,
return self.rotary_emb.forward_3d_sequence_shard(
local_len, rank, frame_stride_local, width_local, device
)
t_idx = token_indices // frame_stride_local
rem = token_indices % frame_stride_local
h_idx = rem // width_local
w_idx = rem % width_local
positions = torch.stack((t_idx, h_idx, w_idx), dim=1)
return self.rotary_emb.forward_uncached(positions)
def forward(
self,
@@ -1379,10 +1379,6 @@ class ZImageTransformer2DModel(CachableDiT, LayerwiseOffloadableModuleMixin):
self._cached_joint_attn_mask_meta = (cache_key, result)
return self._pin_for_active_capture(result)
@staticmethod
def _has_padding(valid_lens: list[int], target_len: int) -> bool:
return any(int(length) < target_len for length in valid_lens)
@staticmethod
def _as_image_list(hidden_states) -> list[torch.Tensor]:
"""Normalize 4D/5D image latents into per-sample tensors."""
@@ -31,6 +31,18 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload im
from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
def get_attention_head_partition(
num_heads: int, num_kv_heads: int, tp_size: int
) -> tuple[int, int]:
"""Partition query heads and shard or replicate KV heads across TP ranks."""
assert num_heads % tp_size == 0
if num_kv_heads >= tp_size:
assert num_kv_heads % tp_size == 0
else:
assert tp_size % num_kv_heads == 0
return num_heads // tp_size, max(1, num_kv_heads // tp_size)
def get_folding_tp_group(config: EncoderConfig):
"""Return the TP group selected for an encoder."""
mode = config.parallel_folding_mode
@@ -218,7 +230,6 @@ class TextEncoder(
# shard conditions left in the root group stays sharded unless the entry
# point is registered; loaders read this and register each name.
_fsdp_forward_methods: tuple[str, ...] = ()
_stacked_params_mapping: list[tuple[str, str, str]] = field(default_factory=list)
_supported_attention_backends: set[AttentionBackendEnum] = (
TextEncoderConfig()._supported_attention_backends
)
@@ -227,7 +238,6 @@ class TextEncoder(
super().__init__()
self.config = config
self._fsdp_shard_conditions = config.arch_config._fsdp_shard_conditions
self._stacked_params_mapping = config.arch_config.stacked_params_mapping
if not self.supported_attention_backends:
raise ValueError(
f"Subclass {self.__class__.__name__} must define _supported_attention_backends"
@@ -40,6 +40,7 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload im
)
from sglang.multimodal_gen.runtime.models.encoders.base import (
EncoderTensorParallelMixin,
get_attention_head_partition,
)
logger = logging.getLogger(__name__)
@@ -114,14 +115,10 @@ class Gemma2Attention(nn.Module):
self.hidden_size = hidden_size
tp_size = get_tp_world_size()
self.total_num_heads = num_heads
assert self.total_num_heads % tp_size == 0
self.num_heads = self.total_num_heads // tp_size
self.total_num_kv_heads = num_kv_heads
if self.total_num_kv_heads >= tp_size:
assert self.total_num_kv_heads % tp_size == 0
else:
assert tp_size % self.total_num_kv_heads == 0
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
self.num_heads, self.num_kv_heads = get_attention_head_partition(
num_heads, num_kv_heads, tp_size
)
arch = config.arch_config
self.head_dim = arch.head_dim
@@ -27,6 +27,7 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload im
)
from sglang.multimodal_gen.runtime.models.encoders.base import (
EncoderTensorParallelMixin,
get_attention_head_partition,
)
from sglang.multimodal_gen.runtime.utils.common import add_prefix
from sglang.srt.models.siglip import SiglipVisionModel
@@ -68,6 +69,36 @@ def get_attention_sliding_window_size(config):
return config.sliding_window - 1
def _load_with_shard_id(
weight_loader, param, loaded_weight: torch.Tensor, shard_id
) -> None:
"""Try the supplied shard ID, then its equivalent q/k/v or integer form."""
try:
weight_loader(param, loaded_weight, shard_id)
return
except (AssertionError, TypeError):
pass
if isinstance(shard_id, str):
mapping = {"q": 0, "k": 1, "v": 2}
if shard_id in mapping:
weight_loader(param, loaded_weight, mapping[shard_id])
return
if shard_id.isdigit():
weight_loader(param, loaded_weight, int(shard_id))
return
elif isinstance(shard_id, int):
mapping = {0: "q", 1: "k", 2: "v"}
if shard_id in mapping:
weight_loader(param, loaded_weight, mapping[shard_id])
return
raise TypeError(
f"Unsupported shard_id={shard_id!r} for weight_loader={weight_loader} "
f"(param={getattr(param, 'name', '<param>')})."
)
class Gemma3RMSNorm(nn.Module):
def __init__(self, dim: int, eps: float = 1e-6):
super().__init__()
@@ -146,14 +177,10 @@ class Gemma3Attention(nn.Module):
self.hidden_size = hidden_size
tp_size = get_tp_world_size()
self.total_num_heads = num_heads
assert self.total_num_heads % tp_size == 0
self.num_heads = self.total_num_heads // tp_size
self.total_num_kv_heads = num_kv_heads
if self.total_num_kv_heads >= tp_size:
assert self.total_num_kv_heads % tp_size == 0
else:
assert tp_size % self.total_num_kv_heads == 0
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
self.num_heads, self.num_kv_heads = get_attention_head_partition(
num_heads, num_kv_heads, tp_size
)
self.head_dim = getattr(
config.text_config, "head_dim", self.hidden_size // self.total_num_heads
@@ -51,10 +51,12 @@ from sglang.multimodal_gen.runtime.layers.vocab_parallel_embedding import (
VocabParallelEmbedding,
)
from sglang.multimodal_gen.runtime.loader.weight_utils import (
load_stacked_weight,
maybe_remap_kv_scale_name,
load_llm_encoder_weights,
)
from sglang.multimodal_gen.runtime.models.encoders.base import (
TextEncoder,
get_attention_head_partition,
)
from sglang.multimodal_gen.runtime.models.encoders.base import TextEncoder
class LlamaMLP(nn.Module):
@@ -116,18 +118,10 @@ class LlamaAttention(nn.Module):
self.hidden_size = hidden_size
tp_size = get_tp_world_size()
self.total_num_heads = num_heads
assert self.total_num_heads % tp_size == 0
self.num_heads = self.total_num_heads // tp_size
self.total_num_kv_heads = num_kv_heads
if self.total_num_kv_heads >= tp_size:
# Number of KV heads is greater than TP size, so we partition
# the KV heads across multiple tensor parallel GPUs.
assert self.total_num_kv_heads % tp_size == 0
else:
# Number of KV heads is less than TP size, so we replicate
# the KV heads across multiple tensor parallel GPUs.
assert tp_size % self.total_num_kv_heads == 0
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
self.num_heads, self.num_kv_heads = get_attention_head_partition(
num_heads, num_kv_heads, tp_size
)
# MistralConfig has an optional head_dim introduced by Mistral-Nemo
self.head_dim = getattr(
config, "head_dim", self.hidden_size // self.total_num_heads
@@ -390,32 +384,11 @@ class LlamaModel(TextEncoder):
return output
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
if "rotary_emb.inv_freq" in name:
continue
if "rotary_emb.cos_cached" in name or "rotary_emb.sin_cached" in name:
# Models trained using ColossalAI may include these tensors in
# the checkpoint. Skip them.
continue
if "scale" in name:
# Remapping the name of FP8 kv-scale.
kv_scale_name: str | None = maybe_remap_kv_scale_name(name, params_dict)
if kv_scale_name is None:
continue
else:
name = kv_scale_name
name = load_stacked_weight(
name,
loaded_weight,
params_dict,
self.config.arch_config.stacked_params_mapping,
)
if name is not None:
loaded_params.add(name)
return loaded_params
return load_llm_encoder_weights(
weights,
dict(self.named_parameters()),
self.config.arch_config.stacked_params_mapping,
)
EntryClass = LlamaModel
@@ -1332,13 +1332,10 @@ class Qwen2_5_VLForConditionalGeneration(TextEncoder):
break
if loaded_stacked_param:
continue
try:
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
param = params_dict[name]
except KeyError:
raise
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
loaded_weight = loaded_weight.to(param.dtype)
@@ -20,10 +20,12 @@ from sglang.multimodal_gen.runtime.layers.vocab_parallel_embedding import (
VocabParallelEmbedding,
)
from sglang.multimodal_gen.runtime.loader.weight_utils import (
load_stacked_weight,
maybe_remap_kv_scale_name,
load_llm_encoder_weights,
)
from sglang.multimodal_gen.runtime.models.encoders.base import (
TextEncoder,
get_attention_head_partition,
)
from sglang.multimodal_gen.runtime.models.encoders.base import TextEncoder
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.layernorm import RMSNorm
@@ -91,14 +93,10 @@ class Qwen3Attention(nn.Module):
self.hidden_size = hidden_size
tp_size = get_tp_world_size()
self.total_num_heads = num_heads
assert self.total_num_heads % tp_size == 0
self.num_heads = self.total_num_heads // tp_size
self.total_num_kv_heads = num_kv_heads
if self.total_num_kv_heads >= tp_size:
assert self.total_num_kv_heads % tp_size == 0
else:
assert tp_size % self.total_num_kv_heads == 0
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
self.num_heads, self.num_kv_heads = get_attention_head_partition(
num_heads, num_kv_heads, tp_size
)
self.head_dim = getattr(
config, "head_dim", self.hidden_size // self.total_num_heads
@@ -432,38 +430,12 @@ class Qwen3ForCausalLM(TextEncoder):
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
"""Load weights with support for tensor parallelism and weight remapping."""
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
# Strip 'model.' prefix from HuggingFace Qwen3 weights
if name.startswith("model."):
name = name[6:] # len("model.") == 6
# Skip rotary embedding weights
if "rotary_emb.inv_freq" in name:
continue
if "rotary_emb.cos_cached" in name or "rotary_emb.sin_cached" in name:
continue
# Handle KV scale remapping
if "scale" in name:
kv_scale_name: str | None = maybe_remap_kv_scale_name(name, params_dict)
if kv_scale_name is None:
continue
else:
name = kv_scale_name
name = load_stacked_weight(
name,
loaded_weight,
params_dict,
self.config.arch_config.stacked_params_mapping,
)
if name is not None:
loaded_params.add(name)
return loaded_params
return load_llm_encoder_weights(
weights,
dict(self.named_parameters()),
self.config.arch_config.stacked_params_mapping,
strip_prefix="model.",
)
EntryClass = Qwen3ForCausalLM
@@ -158,9 +158,6 @@ class T5LayerFF(nn.Module):
# T5 has attn_bias and does not use softmax scaling
class T5MultiHeadAttention(nn.Module):
def __init__(self) -> None:
super().__init__()
def forward(self, q, k, v, attn_bias=None):
b, _, n, c = q.shape
attn = torch.einsum("binc,bjnc->bnij", q, k)
@@ -509,7 +506,6 @@ class T5Stack(nn.Module):
):
super().__init__()
self.embed_tokens = embed_tokens
self.is_umt5 = is_umt5
if is_umt5:
self.block = nn.ModuleList(
[
@@ -49,7 +49,6 @@ class FlowMatchScheduler(BaseScheduler):
self,
num_inference_steps=100,
denoising_strength=1.0,
training=False,
shift=None,
dynamic_shift_len=None,
):
@@ -91,17 +90,6 @@ class FlowMatchScheduler(BaseScheduler):
if self.train_timesteps is None:
self.train_timesteps = self.timesteps
self.train_sigmas = self.sigmas
if training:
x = self.timesteps
y = torch.exp(
-2 * ((x - num_inference_steps / 2) / num_inference_steps) ** 2
)
y_shifted = y - y.min()
bsmntw_weighing = y_shifted * (num_inference_steps / y_shifted.sum())
self.linear_timesteps_weights = bsmntw_weighing
self.training = True
else:
self.training = False
def scale_model_input(self, sample: torch.Tensor, timestep: int | None = None):
return sample
@@ -118,14 +106,6 @@ class FlowMatchScheduler(BaseScheduler):
prev_sample = sample + model_output * (sigma_ - sigma)
return prev_sample
def return_to_timestep(self, timestep, sample, sample_stablized):
if isinstance(timestep, torch.Tensor):
timestep = timestep.cpu()
timestep_id = torch.argmin((self.timesteps - timestep).abs())
sigma = self.sigmas[timestep_id]
model_output = (sample - sample_stablized) / sigma
return model_output
def add_noise(self, original_samples, noise, timestep):
if isinstance(timestep, torch.Tensor):
timestep = timestep.cpu()
@@ -134,17 +114,6 @@ class FlowMatchScheduler(BaseScheduler):
sample = (1 - sigma) * original_samples + sigma * noise
return sample
def training_target(self, sample, noise, timestep):
target = noise - sample
return target
def training_weight(self, timestep):
timestep_id = torch.argmin(
(self.timesteps - timestep.to(self.timesteps.device)).abs()
)
weights = self.linear_timesteps_weights[timestep_id]
return weights
def calculate_shift(
self,
image_seq_len,
@@ -268,9 +237,9 @@ class FlowMatchPairScheduler(FlowMatchScheduler):
self.set_pair_postprocess(_quadratic_perp_bulge_swap)
return
if name == "v2a_sequential":
if name in ("v2a_sequential", "a2v_sequential"):
def _v2a(pairs: torch.Tensor):
def _sequential(pairs: torch.Tensor):
if (
not isinstance(pairs, torch.Tensor)
or pairs.ndim != 2
@@ -281,8 +250,12 @@ class FlowMatchPairScheduler(FlowMatchScheduler):
base = pairs[:, 0]
seq_half = base[::2]
m = int(seq_half.shape[0])
col0 = torch.cat([seq_half, seq_half[-1:].repeat(m)], dim=0)[:N]
col1 = torch.cat([seq_half[0:1].repeat(m), seq_half], dim=0)[:N]
if name == "v2a_sequential":
col0 = torch.cat([seq_half, seq_half[-1:].repeat(m)], dim=0)[:N]
col1 = torch.cat([seq_half[0:1].repeat(m), seq_half], dim=0)[:N]
else:
col0 = torch.cat([seq_half[0:1].repeat(m), seq_half], dim=0)[:N]
col1 = torch.cat([seq_half, seq_half[-1:].repeat(m)], dim=0)[:N]
return torch.stack(
[
col0.to(dtype=pairs.dtype, device=pairs.device),
@@ -291,60 +264,25 @@ class FlowMatchPairScheduler(FlowMatchScheduler):
dim=1,
)
self.set_pair_postprocess(_v2a)
self.set_pair_postprocess(_sequential)
return
if name == "a2v_sequential":
if name in ("v2a", "a2v"):
def _a2v(pairs: torch.Tensor):
def _single_modality(pairs: torch.Tensor):
if (
not isinstance(pairs, torch.Tensor)
or pairs.ndim != 2
or pairs.shape[1] != 2
):
raise ValueError("pairs must be a torch.Tensor of shape [N, 2]")
N = pairs.shape[0]
base = pairs[:, 0]
seq_half = base[::2]
m = int(seq_half.shape[0])
col0 = torch.cat([seq_half[0:1].repeat(m), seq_half], dim=0)[:N]
col1 = torch.cat([seq_half, seq_half[-1:].repeat(m)], dim=0)[:N]
return torch.stack(
[
col0.to(dtype=pairs.dtype, device=pairs.device),
col1.to(dtype=pairs.dtype, device=pairs.device),
],
dim=1,
)
if name == "v2a":
zeros = torch.zeros_like(pairs[:, 0])
return torch.stack([zeros, pairs[:, 1]], dim=1)
else:
zeros = torch.zeros_like(pairs[:, 1])
return torch.stack([pairs[:, 0], zeros], dim=1)
self.set_pair_postprocess(_a2v)
return
if name == "v2a":
def _v2a_classic(pairs: torch.Tensor):
if (
not isinstance(pairs, torch.Tensor)
or pairs.ndim != 2
or pairs.shape[1] != 2
):
raise ValueError("pairs must be a torch.Tensor of shape [N, 2]")
zeros = torch.zeros_like(pairs[:, 0])
return torch.stack([zeros, pairs[:, 1]], dim=1)
self.set_pair_postprocess(_v2a_classic)
return
if name == "a2v":
def _a2v_classic(pairs: torch.Tensor):
if (
not isinstance(pairs, torch.Tensor)
or pairs.ndim != 2
or pairs.shape[1] != 2
):
raise ValueError("pairs must be a torch.Tensor of shape [N, 2]")
zeros = torch.zeros_like(pairs[:, 1])
return torch.stack([pairs[:, 0], zeros], dim=1)
self.set_pair_postprocess(_a2v_classic)
self.set_pair_postprocess(_single_modality)
return
if name == "dual_sigma_shift":
visual_shift = float(kwargs.get("visual_shift", self.shift))
@@ -19,21 +19,7 @@ import torch
import torch.nn as nn
import torch.nn.functional as F
if "sinc" in dir(torch):
sinc = torch.sinc
else:
# This code is adopted from adefossez's julius.core.sinc under the MIT License
# https://adefossez.github.io/julius/julius/core.html
def sinc(x: torch.Tensor):
"""
Implementation of sinc, i.e. sin(pi * x) / (pi * x)
__Warning__: Different to julius.sinc, the input is multiplied by `pi`!
"""
return torch.where(
x == 0,
torch.tensor(1.0, device=x.device, dtype=x.dtype),
torch.sin(math.pi * x) / math.pi / x,
)
sinc = torch.sinc
# This code is adopted from adefossez's julius.lowpass.LowPassFilters under the MIT License
@@ -175,8 +161,6 @@ class Activation1d(nn.Module):
down_kernel_size: int = 12,
):
super().__init__()
self.up_ratio = up_ratio
self.down_ratio = down_ratio
self.act = activation
self.upsample = UpSample1d(up_ratio, up_kernel_size)
self.downsample = DownSample1d(down_ratio, down_kernel_size)
@@ -951,18 +951,6 @@ class AutoencoderKLQwenImage(ParallelTiledVAE):
self.tile_sample_stride_height = 192
self.tile_sample_stride_width = 192
# Precompute and cache conv counts for encoder and decoder for clear_cache speedup
self._cached_conv_counts = {
"decoder": sum(
isinstance(m, (QwenImageCausalConv3d, SpatialParallelCausalConv3d))
for m in self.decoder.modules()
)
if self.decoder is not None
else 0,
"encoder": sum(isinstance(m, QwenImageCausalConv3d) for m in self.encoder.modules())
if self.encoder is not None
else 0,
}
cuda_device = get_local_torch_device()
dtype = torch.get_default_dtype()
latent_channels = config.arch_config.z_dim
@@ -7,7 +7,6 @@ from functools import lru_cache
from math import isqrt, prod
from typing import Optional, cast
import numpy as np
import torch
import torch.distributed as dist
from diffusers.models.autoencoders.vae import DiagonalGaussianDistribution
@@ -230,6 +229,14 @@ class ParallelTiledVAE(ABC, nn.Module, LayerwiseOffloadableModuleMixin):
) + b[:, :, x, :, :] * (x / blend_extent)
return b
def _spatial_tile_geometry(self) -> tuple[int, int, int, int]:
return (
self.tile_sample_min_height // self.spatial_compression_ratio,
self.tile_sample_min_width // self.spatial_compression_ratio,
self.tile_sample_stride_height // self.spatial_compression_ratio,
self.tile_sample_stride_width // self.spatial_compression_ratio,
)
def spatial_tiled_encode(self, x: torch.Tensor) -> torch.Tensor:
r"""Encode a batch of images using a tiled encoder.
@@ -244,18 +251,12 @@ class ParallelTiledVAE(ABC, nn.Module, LayerwiseOffloadableModuleMixin):
# latent_height = height // self.spatial_compression_ratio
# latent_width = width // self.spatial_compression_ratio
tile_latent_min_height = (
self.tile_sample_min_height // self.spatial_compression_ratio
)
tile_latent_min_width = (
self.tile_sample_min_width // self.spatial_compression_ratio
)
tile_latent_stride_height = (
self.tile_sample_stride_height // self.spatial_compression_ratio
)
tile_latent_stride_width = (
self.tile_sample_stride_width // self.spatial_compression_ratio
)
(
tile_latent_min_height,
tile_latent_min_width,
tile_latent_stride_height,
tile_latent_stride_width,
) = self._spatial_tile_geometry()
blend_height = tile_latent_min_height - tile_latent_stride_height
blend_width = tile_latent_min_width - tile_latent_stride_width
@@ -293,21 +294,15 @@ class ParallelTiledVAE(ABC, nn.Module, LayerwiseOffloadableModuleMixin):
world_size, rank = sp_group.world_size, sp_group.rank_in_group
_, _, T, H, W = z.shape
tile_latent_min_height = (
self.tile_sample_min_height // self.spatial_compression_ratio
)
tile_latent_min_width = (
self.tile_sample_min_width // self.spatial_compression_ratio
)
(
tile_latent_min_height,
tile_latent_min_width,
tile_latent_stride_height,
tile_latent_stride_width,
) = self._spatial_tile_geometry()
tile_latent_min_num_frames = (
self.tile_sample_min_num_frames // self.temporal_compression_ratio
)
tile_latent_stride_height = (
self.tile_sample_stride_height // self.spatial_compression_ratio
)
tile_latent_stride_width = (
self.tile_sample_stride_width // self.spatial_compression_ratio
)
tile_latent_stride_num_frames = (
self.tile_sample_stride_num_frames // self.temporal_compression_ratio
)
@@ -436,18 +431,12 @@ class ParallelTiledVAE(ABC, nn.Module, LayerwiseOffloadableModuleMixin):
if world_size <= 1:
return self._decode(z)
tile_latent_min_height = (
self.tile_sample_min_height // self.spatial_compression_ratio
)
tile_latent_min_width = (
self.tile_sample_min_width // self.spatial_compression_ratio
)
tile_latent_stride_height = (
self.tile_sample_stride_height // self.spatial_compression_ratio
)
tile_latent_stride_width = (
self.tile_sample_stride_width // self.spatial_compression_ratio
)
(
tile_latent_min_height,
tile_latent_min_width,
tile_latent_stride_height,
tile_latent_stride_width,
) = self._spatial_tile_geometry()
overlap_h = max(0, tile_latent_min_height - tile_latent_stride_height)
overlap_w = max(0, tile_latent_min_width - tile_latent_stride_width)
halo_h = overlap_h // 2
@@ -574,18 +563,12 @@ class ParallelTiledVAE(ABC, nn.Module, LayerwiseOffloadableModuleMixin):
# sample_height = height * self.spatial_compression_ratio
# sample_width = width * self.spatial_compression_ratio
tile_latent_min_height = (
self.tile_sample_min_height // self.spatial_compression_ratio
)
tile_latent_min_width = (
self.tile_sample_min_width // self.spatial_compression_ratio
)
tile_latent_stride_height = (
self.tile_sample_stride_height // self.spatial_compression_ratio
)
tile_latent_stride_width = (
self.tile_sample_stride_width // self.spatial_compression_ratio
)
(
tile_latent_min_height,
tile_latent_min_width,
tile_latent_stride_height,
tile_latent_stride_width,
) = self._spatial_tile_geometry()
blend_height = self.tile_sample_min_height - self.tile_sample_stride_height
blend_width = self.tile_sample_min_width - self.tile_sample_stride_width
@@ -809,16 +792,5 @@ class DiagonalGaussianDistribution:
dim=dims,
)
def nll(
self, sample: torch.Tensor, dims: tuple[int, ...] = (1, 2, 3)
) -> torch.Tensor:
if self.deterministic:
return torch.Tensor([0.0])
logtwopi = np.log(2.0 * np.pi)
return 0.5 * torch.sum(
logtwopi + self.logvar + torch.pow(sample - self.mean, 2) / self.var,
dim=dims,
)
def mode(self) -> torch.Tensor:
return self.mean
@@ -193,12 +193,6 @@ class Cosmos3AVAEAudioTokenizer(nn.Module, LayerwiseOffloadableModuleMixin):
def temporal_compression_factor(self) -> int:
return self.hop_size
def get_latent_num_samples(self, num_audio_samples: int) -> int:
return int(num_audio_samples) // self.hop_size
def get_audio_num_samples(self, num_latent_samples: int) -> int:
return int(num_latent_samples) * self.hop_size
def _denormalize_latent(self, latent: torch.Tensor) -> torch.Tensor:
if self.normalization_type == "tanh":
in_dtype = latent.dtype
@@ -370,7 +370,6 @@ class Encoder(nn.Module):
# Wrap black into nn.Sequential
self.block = nn.Sequential(*self.block)
self.enc_dim = d_model
def forward(self, x):
return self.block(x)
@@ -209,7 +209,6 @@ class QKVMultiheadCrossAttention(nn.Module):
):
super().__init__()
self.heads = heads
self.n_data = n_data
self.q_norm = (
norm_layer(width // heads, elementwise_affine=True, eps=1e-6)
if qk_norm
@@ -255,7 +254,6 @@ class MultiheadCrossAttention(nn.Module):
kv_cache: bool = False,
):
super().__init__()
self.n_data = n_data
self.width = width
self.heads = heads
self.data_width = width if data_width is None else data_width
@@ -338,7 +336,6 @@ class QKVMultiheadAttention(nn.Module):
):
super().__init__()
self.heads = heads
self.n_ctx = n_ctx
self.q_norm = (
norm_layer(width // heads, elementwise_affine=True, eps=1e-6)
if qk_norm
@@ -381,7 +378,6 @@ class MultiheadAttention(nn.Module):
drop_path_rate: float = 0.0,
):
super().__init__()
self.n_ctx = n_ctx
self.width = width
self.heads = heads
self.c_qkv = nn.Linear(width, width * 3, bias=qkv_bias)
@@ -450,7 +446,6 @@ class Transformer(nn.Module):
drop_path_rate: float = 0.0,
):
super().__init__()
self.n_ctx = n_ctx
self.width = width
self.layers = layers
self.resblocks = nn.ModuleList(
@@ -512,7 +507,6 @@ class CrossAttentionDecoder(nn.Module):
if self.enable_ln_post:
self.ln_post = nn.LayerNorm(width)
self.output_proj = nn.Linear(width, out_channels)
self.label_type = label_type
def set_cross_attention_processor(self, processor):
self.cross_attn_decoder.attn.attention.attn_processor = processor
@@ -131,7 +131,6 @@ class LTX23VideoConditionEncoder(
latent_log_var = str(vae_config.get("latent_log_var", "uniform"))
self.patch_size = patch_size
self.latency_channels = latent_channels
self.latent_log_var = latent_log_var
self.per_channel_statistics = LTX23PerChannelStatistics(latent_channels)

Some files were not shown because too many files have changed in this diff Show More