[diffusion] refactor: remove unused helpers and write-only state (#43064)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
@@ -3,14 +3,11 @@ SGLang Diffusion Server API client.
|
||||
Provides a low-level interface for interacting with SGLang Diffusion HTTP server.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import io
|
||||
import os
|
||||
import time
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
import requests
|
||||
from PIL import Image
|
||||
|
||||
|
||||
class SGLDiffusionServerAPI:
|
||||
@@ -417,38 +414,6 @@ class SGLDiffusionServerAPI:
|
||||
}
|
||||
return content_types.get(ext, "image/png")
|
||||
|
||||
def decode_image_from_response(
|
||||
self, response_data: Dict[str, Any], index: int = 0
|
||||
) -> Image.Image:
|
||||
"""
|
||||
Decode base64 image from API response.
|
||||
|
||||
Args:
|
||||
response_data: API response dictionary
|
||||
index: Index of the image in the response (default: 0)
|
||||
|
||||
Returns:
|
||||
PIL Image object
|
||||
"""
|
||||
if "data" not in response_data or not response_data["data"]:
|
||||
raise ValueError("No image data in response")
|
||||
|
||||
if index >= len(response_data["data"]):
|
||||
raise IndexError(f"Image index {index} out of range")
|
||||
|
||||
image_data = response_data["data"][index]
|
||||
if "b64_json" not in image_data or not image_data["b64_json"]:
|
||||
raise ValueError("No base64 image data found")
|
||||
|
||||
image_bytes = base64.b64decode(image_data["b64_json"])
|
||||
image = Image.open(io.BytesIO(image_bytes))
|
||||
|
||||
# Convert to RGB if needed
|
||||
if image.mode != "RGB":
|
||||
image = image.convert("RGB")
|
||||
|
||||
return image
|
||||
|
||||
def set_lora(
|
||||
self,
|
||||
lora_nickname: str,
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import argparse
|
||||
import os
|
||||
|
||||
from sglang.multimodal_gen.apps.webui.minimax_h3 import (
|
||||
@@ -21,13 +20,6 @@ from sglang.srt.environ import envs
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def add_webui_args(parser: argparse.ArgumentParser):
|
||||
"""Add the arguments for the generate command."""
|
||||
parser = ServerArgs.add_cli_args(parser)
|
||||
parser = SamplingParams.add_cli_args(parser)
|
||||
return parser
|
||||
|
||||
|
||||
def run_sgl_diffusion_webui(server_args: ServerArgs):
|
||||
if is_minimax_h3(server_args):
|
||||
return run_minimax_h3_webui(server_args)
|
||||
|
||||
@@ -65,11 +65,6 @@ class Krea2ArchConfig(DiTArchConfig):
|
||||
def head_dim(self) -> int:
|
||||
return self.features // self.heads
|
||||
|
||||
@property
|
||||
def in_features_packed(self) -> int:
|
||||
"""Patch-embed input width: channels * patch**2."""
|
||||
return self.channels * self.patch**2
|
||||
|
||||
|
||||
@dataclass
|
||||
class Krea2DitConfig(DiTConfig):
|
||||
|
||||
@@ -17,12 +17,6 @@ class LTXModelType(Enum):
|
||||
VideoOnly = "ltx video only model"
|
||||
AudioOnly = "ltx audio only model"
|
||||
|
||||
def is_video_enabled(self) -> bool:
|
||||
return self in (LTXModelType.AudioVideo, LTXModelType.VideoOnly)
|
||||
|
||||
def is_audio_enabled(self) -> bool:
|
||||
return self in (LTXModelType.AudioVideo, LTXModelType.AudioOnly)
|
||||
|
||||
|
||||
class LTX2RopeType(str, Enum):
|
||||
"""
|
||||
|
||||
@@ -31,7 +31,4 @@ class Ideogram4TextEncoderConfig(Qwen3VLConfig):
|
||||
self.arch_config.ideogram_fp8_weight_only = not use_bitsandbytes
|
||||
self.arch_config.requires_gpu_resident_text_encoder = use_bitsandbytes
|
||||
|
||||
def finalize_model_arch(self):
|
||||
self.post_diffusers_config_update()
|
||||
|
||||
arch_config: Qwen3VLArchConfig = field(default_factory=Qwen3VLArchConfig)
|
||||
|
||||
@@ -16,6 +16,19 @@ AUTO_PARALLEL_DECODE_MODE = "auto"
|
||||
SPATIAL_SHARD_PARALLEL_DECODE_MODES = ("spatial_shard", "spatial")
|
||||
|
||||
|
||||
def get_channelwise_decode_scale_and_shift(arch_config, device, dtype):
|
||||
"""Return reciprocal std and mean broadcast over BCTHW latents."""
|
||||
scaling_factor = 1.0 / torch.tensor(arch_config.latents_std, device=device).view(
|
||||
1, arch_config.z_dim, 1, 1, 1
|
||||
).to(device, dtype)
|
||||
shift_factor = (
|
||||
torch.tensor(arch_config.latents_mean)
|
||||
.view(1, arch_config.z_dim, 1, 1, 1)
|
||||
.to(device, dtype)
|
||||
)
|
||||
return scaling_factor, shift_factor
|
||||
|
||||
|
||||
@lru_cache(maxsize=8)
|
||||
def is_spatial_shard_parallel_decode_mode(mode: str) -> bool:
|
||||
return mode in SPATIAL_SHARD_PARALLEL_DECODE_MODES
|
||||
|
||||
@@ -2,9 +2,38 @@
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, List
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.configs.models.vaes.base import VAEArchConfig, VAEConfig
|
||||
|
||||
|
||||
def get_ltx_video_decode_scale_and_shift(device, dtype, vae, arch_config):
|
||||
"""Invert LTX video normalization: (z - mean) * scaling_factor / std."""
|
||||
latents_mean = getattr(vae, "latents_mean", None)
|
||||
latents_std = getattr(vae, "latents_std", None)
|
||||
scaling_factor = (
|
||||
getattr(getattr(vae, "config", None), "scaling_factor", None)
|
||||
or getattr(vae, "scaling_factor", None)
|
||||
or getattr(arch_config, "scaling_factor", None)
|
||||
or 1.0
|
||||
)
|
||||
if isinstance(scaling_factor, (int, float)) and float(scaling_factor) == 0.0:
|
||||
scaling_factor = 1.0
|
||||
|
||||
if isinstance(latents_mean, torch.Tensor) and isinstance(latents_std, torch.Tensor):
|
||||
latents_mean = latents_mean.to(device=device, dtype=dtype).view(1, -1, 1, 1, 1)
|
||||
latents_std = latents_std.to(device=device, dtype=dtype).view(1, -1, 1, 1, 1)
|
||||
sf = torch.tensor(float(scaling_factor), device=device, dtype=dtype).view(
|
||||
1, 1, 1, 1, 1
|
||||
)
|
||||
return sf / latents_std, latents_mean
|
||||
|
||||
sf = torch.tensor(float(scaling_factor), device=device, dtype=dtype).view(
|
||||
1, 1, 1, 1, 1
|
||||
)
|
||||
return sf, None
|
||||
|
||||
|
||||
@dataclass
|
||||
class LTXVideoVAEArchConfig(VAEArchConfig):
|
||||
# Architecture params
|
||||
|
||||
@@ -5,9 +5,8 @@ import json
|
||||
import math
|
||||
import os
|
||||
from collections.abc import Callable
|
||||
from dataclasses import asdict, dataclass, field, fields
|
||||
from dataclasses import dataclass, field, fields
|
||||
from enum import Enum
|
||||
from operator import attrgetter
|
||||
from typing import Any, ClassVar
|
||||
|
||||
import numpy as np
|
||||
@@ -56,6 +55,16 @@ class STA_Mode(str, Enum):
|
||||
NONE = None
|
||||
|
||||
|
||||
def pack_latents_2x2(latents, batch_size, num_channels_latents, height, width):
|
||||
latents = latents.view(
|
||||
batch_size, num_channels_latents, height // 2, 2, width // 2, 2
|
||||
)
|
||||
latents = latents.permute(0, 2, 4, 1, 3, 5)
|
||||
return latents.reshape(
|
||||
batch_size, (height // 2) * (width // 2), num_channels_latents * 4
|
||||
)
|
||||
|
||||
|
||||
def postprocess_text(output: BaseEncoderOutput, _text_inputs) -> torch.tensor:
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -1198,35 +1207,6 @@ class PipelineConfig:
|
||||
f"Length of text postprocess functions ({len(self.postprocess_text_funcs)}) must be equal to length of text preprocessing functions ({len(self.preprocess_text_funcs)})"
|
||||
)
|
||||
|
||||
def dump_to_json(self, file_path: str):
|
||||
output_dict = {f.name: attrgetter(f.name)(self) for f in fields(self)}
|
||||
del_keys = []
|
||||
for key, value in output_dict.items():
|
||||
if isinstance(value, ModelConfig):
|
||||
model_dict = asdict(value)
|
||||
# Model Arch Config should be hidden away from the users
|
||||
model_dict.pop("arch_config")
|
||||
output_dict[key] = model_dict
|
||||
elif isinstance(value, tuple) and all(
|
||||
isinstance(v, ModelConfig) for v in value
|
||||
):
|
||||
model_dicts = []
|
||||
for v in value:
|
||||
model_dict = asdict(v)
|
||||
# Model Arch Config should be hidden away from the users
|
||||
model_dict.pop("arch_config")
|
||||
model_dicts.append(model_dict)
|
||||
output_dict[key] = model_dicts
|
||||
elif isinstance(value, tuple) and all(callable(f) for f in value):
|
||||
# Skip dumping functions
|
||||
del_keys.append(key)
|
||||
|
||||
for key in del_keys:
|
||||
output_dict.pop(key, None)
|
||||
|
||||
with open(file_path, "w") as f:
|
||||
json.dump(output_dict, f, indent=2)
|
||||
|
||||
def load_from_json(self, file_path: str):
|
||||
with open(file_path) as f:
|
||||
input_pipeline_dict = json.load(f)
|
||||
|
||||
@@ -67,12 +67,6 @@ class DiffusersGenericPipelineConfig(PipelineConfig):
|
||||
"""
|
||||
pass
|
||||
|
||||
def adjust_size(self, width, height, image):
|
||||
"""
|
||||
Pass through - diffusers handles size adjustments.
|
||||
"""
|
||||
return width, height
|
||||
|
||||
|
||||
# Static subclasses for each non-default task type.
|
||||
# These exist so that _get_diffusers_model_info() can swap the task_type without
|
||||
|
||||
@@ -20,12 +20,12 @@ from sglang.multimodal_gen.configs.models.vaes.flux import Flux2VAEConfig, FluxV
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.base import (
|
||||
ImagePipelineConfig,
|
||||
ModelTaskType,
|
||||
pack_latents_2x2,
|
||||
shard_rotary_emb_for_sp,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.hunyuan import (
|
||||
clip_postprocess_text,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.qwen_image import _pack_latents
|
||||
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
|
||||
|
||||
|
||||
@@ -141,7 +141,9 @@ class FluxPipelineConfig(ImagePipelineConfig):
|
||||
width = 2 * (batch.width // (self.vae_config.arch_config.vae_scale_factor * 2))
|
||||
num_channels_latents = self.dit_config.arch_config.in_channels // 4
|
||||
# pack latents
|
||||
return _pack_latents(latents, batch_size, num_channels_latents, height, width)
|
||||
return pack_latents_2x2(
|
||||
latents, batch_size, num_channels_latents, height, width
|
||||
)
|
||||
|
||||
def get_pos_prompt_embeds(self, batch):
|
||||
return batch.prompt_embeds[1]
|
||||
|
||||
@@ -11,6 +11,9 @@ from sglang.multimodal_gen.configs.models import DiTConfig, EncoderConfig, VAECo
|
||||
from sglang.multimodal_gen.configs.models.dits.joy_image import JoyImageDiTConfig
|
||||
from sglang.multimodal_gen.configs.models.encoders.qwen3vl import Qwen3VLConfig
|
||||
from sglang.multimodal_gen.configs.models.vaes import WanVAEConfig
|
||||
from sglang.multimodal_gen.configs.models.vaes.base import (
|
||||
get_channelwise_decode_scale_and_shift,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.base import (
|
||||
ImagePipelineConfig,
|
||||
ModelTaskType,
|
||||
@@ -249,32 +252,10 @@ class JoyImageEditPipelineConfig(ImagePipelineConfig):
|
||||
def get_decode_scale_and_shift(
|
||||
self, device, dtype, vae
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Get VAE denormalization scale and shift.
|
||||
|
||||
Args:
|
||||
device: Target device
|
||||
dtype: Target dtype
|
||||
vae: VAE model
|
||||
|
||||
Returns:
|
||||
Tuple of (scaling_factor, shift_factor)
|
||||
"""
|
||||
vae_arch_config = self.vae_config.arch_config
|
||||
|
||||
# Create scale factor: 1.0 / std
|
||||
scaling_factor = 1.0 / torch.tensor(
|
||||
vae_arch_config.latents_std, device=device
|
||||
).view(1, vae_arch_config.z_dim, 1, 1, 1).to(device, dtype)
|
||||
|
||||
# Create shift factor: mean
|
||||
shift_factor = (
|
||||
torch.tensor(vae_arch_config.latents_mean)
|
||||
.view(1, vae_arch_config.z_dim, 1, 1, 1)
|
||||
.to(device, dtype)
|
||||
return get_channelwise_decode_scale_and_shift(
|
||||
self.vae_config.arch_config, device, dtype
|
||||
)
|
||||
|
||||
return scaling_factor, shift_factor
|
||||
|
||||
def prepare_calculated_size(self, img: Image.Image) -> Tuple[int, int]:
|
||||
img_h, img_w = img.size[1], img.size[0] # PIL (w,h)
|
||||
bucket = self.find_best_bucket((1, 1, img_h, img_w))
|
||||
|
||||
@@ -6,6 +6,9 @@ from diffusers.image_processor import VaeImageProcessor
|
||||
|
||||
from sglang.multimodal_gen.configs.models import DiTConfig, VAEConfig
|
||||
from sglang.multimodal_gen.configs.models.dits.krea2 import Krea2DitConfig
|
||||
from sglang.multimodal_gen.configs.models.vaes.base import (
|
||||
get_channelwise_decode_scale_and_shift,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.models.vaes.qwenimage import QwenImageVAEConfig
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.base import (
|
||||
ImagePipelineConfig,
|
||||
@@ -146,16 +149,9 @@ class Krea2PipelineConfig(ImagePipelineConfig):
|
||||
|
||||
# --- VAE decode (same as Qwen-Image: latents * std + mean) ---
|
||||
def get_decode_scale_and_shift(self, device, dtype, vae):
|
||||
vae_arch = self.vae_config.arch_config
|
||||
scaling_factor = 1.0 / torch.tensor(vae_arch.latents_std, device=device).view(
|
||||
1, vae_arch.z_dim, 1, 1, 1
|
||||
).to(device, dtype)
|
||||
shift_factor = (
|
||||
torch.tensor(vae_arch.latents_mean)
|
||||
.view(1, vae_arch.z_dim, 1, 1, 1)
|
||||
.to(device, dtype)
|
||||
return get_channelwise_decode_scale_and_shift(
|
||||
self.vae_config.arch_config, device, dtype
|
||||
)
|
||||
return scaling_factor, shift_factor
|
||||
|
||||
def post_denoising_loop(self, latents, batch):
|
||||
latents, batch_size, channels, height, width = self._unpad_and_unpack_latents(
|
||||
|
||||
@@ -17,6 +17,7 @@ from sglang.multimodal_gen.configs.pipeline_configs.base import (
|
||||
ImagePipelineConfig,
|
||||
ModelTaskType,
|
||||
TextConditioningOutput,
|
||||
pack_latents_2x2,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.model_deployment_config import (
|
||||
ModelDeploymentConfig,
|
||||
@@ -171,17 +172,6 @@ def _unpack_latents(latents, height, width, vae_scale_factor):
|
||||
return latents
|
||||
|
||||
|
||||
def _pack_latents(latents, batch_size, num_channels_latents, height, width):
|
||||
latents = latents.view(
|
||||
batch_size, num_channels_latents, height // 2, 2, width // 2, 2
|
||||
)
|
||||
latents = latents.permute(0, 2, 4, 1, 3, 5)
|
||||
latents = latents.reshape(
|
||||
batch_size, (height // 2) * (width // 2), num_channels_latents * 4
|
||||
)
|
||||
return latents
|
||||
|
||||
|
||||
def _calculate_shift(
|
||||
image_seq_len,
|
||||
base_seq_len: int = 256,
|
||||
@@ -293,7 +283,7 @@ class LongCatImagePipelineConfig(ImagePipelineConfig):
|
||||
def maybe_pack_latents(self, latents, batch_size, batch):
|
||||
num_channels_latents = self.dit_config.arch_config.num_channels_latents
|
||||
_, _, h, w = latents.shape
|
||||
return _pack_latents(latents, batch_size, num_channels_latents, h, w)
|
||||
return pack_latents_2x2(latents, batch_size, num_channels_latents, h, w)
|
||||
|
||||
def maybe_prepare_latent_ids(self, latents):
|
||||
# latents shape after packing: [B, (h//2)*(w//2), C*4]
|
||||
@@ -517,7 +507,7 @@ class LongCatImageEditPipelineConfig(LongCatImagePipelineConfig):
|
||||
batch_size // latent_condition.shape[0], 1, 1, 1
|
||||
)
|
||||
_, num_channels_latents, height, width = latent_condition.shape
|
||||
return _pack_latents(
|
||||
return pack_latents_2x2(
|
||||
latent_condition, batch_size, num_channels_latents, height, width
|
||||
)
|
||||
|
||||
|
||||
@@ -11,7 +11,10 @@ from sglang.multimodal_gen.configs.models.encoders import (
|
||||
)
|
||||
from sglang.multimodal_gen.configs.models.encoders.gemma_3 import Gemma3Config
|
||||
from sglang.multimodal_gen.configs.models.vaes.ltx_audio import LTXAudioVAEConfig
|
||||
from sglang.multimodal_gen.configs.models.vaes.ltx_video import LTXVideoVAEConfig
|
||||
from sglang.multimodal_gen.configs.models.vaes.ltx_video import (
|
||||
LTXVideoVAEConfig,
|
||||
get_ltx_video_decode_scale_and_shift,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.base import (
|
||||
ModelTaskType,
|
||||
PipelineConfig,
|
||||
@@ -564,36 +567,9 @@ class LTX2PipelineConfig(PipelineConfig):
|
||||
)
|
||||
|
||||
def get_decode_scale_and_shift(self, device, dtype, vae):
|
||||
latents_mean = getattr(vae, "latents_mean", None)
|
||||
latents_std = getattr(vae, "latents_std", None)
|
||||
|
||||
scaling_factor = (
|
||||
getattr(getattr(vae, "config", None), "scaling_factor", None)
|
||||
or getattr(vae, "scaling_factor", None)
|
||||
or getattr(self.vae_config.arch_config, "scaling_factor", None)
|
||||
or 1.0
|
||||
return get_ltx_video_decode_scale_and_shift(
|
||||
device, dtype, vae, self.vae_config.arch_config
|
||||
)
|
||||
if isinstance(scaling_factor, (int, float)) and float(scaling_factor) == 0.0:
|
||||
scaling_factor = 1.0
|
||||
|
||||
if isinstance(latents_mean, torch.Tensor) and isinstance(
|
||||
latents_std, torch.Tensor
|
||||
):
|
||||
latents_mean = latents_mean.to(device=device, dtype=dtype).view(
|
||||
1, -1, 1, 1, 1
|
||||
)
|
||||
latents_std = latents_std.to(device=device, dtype=dtype).view(
|
||||
1, -1, 1, 1, 1
|
||||
)
|
||||
sf = torch.tensor(float(scaling_factor), device=device, dtype=dtype).view(
|
||||
1, 1, 1, 1, 1
|
||||
)
|
||||
return sf / latents_std, latents_mean
|
||||
|
||||
sf = torch.tensor(float(scaling_factor), device=device, dtype=dtype).view(
|
||||
1, 1, 1, 1, 1
|
||||
)
|
||||
return sf, None
|
||||
|
||||
@staticmethod
|
||||
def _unpack_latents(
|
||||
@@ -626,29 +602,6 @@ class LTX2PipelineConfig(PipelineConfig):
|
||||
)
|
||||
return latents
|
||||
|
||||
@staticmethod
|
||||
def _denormalize_latents(
|
||||
latents: torch.Tensor,
|
||||
latents_mean: torch.Tensor,
|
||||
latents_std: torch.Tensor,
|
||||
scaling_factor: float = 1.0,
|
||||
) -> torch.Tensor:
|
||||
# Denormalize latents across the channel dimension [B, C, F, H, W]
|
||||
latents_mean = latents_mean.view(1, -1, 1, 1, 1).to(
|
||||
latents.device, latents.dtype
|
||||
)
|
||||
latents_std = latents_std.view(1, -1, 1, 1, 1).to(latents.device, latents.dtype)
|
||||
latents = latents * latents_std / scaling_factor + latents_mean
|
||||
return latents
|
||||
|
||||
@staticmethod
|
||||
def _denormalize_audio_latents(
|
||||
latents: torch.Tensor, latents_mean: torch.Tensor, latents_std: torch.Tensor
|
||||
):
|
||||
latents_mean = latents_mean.to(latents.device, latents.dtype)
|
||||
latents_std = latents_std.to(latents.device, latents.dtype)
|
||||
return (latents * latents_std) + latents_mean
|
||||
|
||||
@staticmethod
|
||||
def _unpack_audio_latents(
|
||||
latents: torch.Tensor,
|
||||
|
||||
@@ -70,16 +70,9 @@ class MiniMaxH3PipelineConfig(PipelineConfig):
|
||||
output_audio_channels: int | None = 2
|
||||
output_av_drift_tolerance_s: float | None = 0.25
|
||||
|
||||
def accepts_audio_input(self) -> bool:
|
||||
return True
|
||||
|
||||
def supports_disaggregation(self) -> bool:
|
||||
return False
|
||||
|
||||
@property
|
||||
def requires_audio_output(self) -> bool:
|
||||
return True
|
||||
|
||||
def get_model_deployment_config(self) -> ModelDeploymentConfig:
|
||||
return ModelDeploymentConfig(
|
||||
speed_mode_enable_torch_compile_by_default=False,
|
||||
|
||||
@@ -154,19 +154,6 @@ class MOVAPipelineConfig(PipelineConfig):
|
||||
latent_T = (num_samples + audio_vae.hop_length - 1) // audio_vae.hop_length
|
||||
return (batch_size, audio_vae.latent_dim, latent_T)
|
||||
|
||||
def normalize_video_latents(self, latents: torch.Tensor, video_vae) -> torch.Tensor:
|
||||
latents_mean = getattr(video_vae.config, "latents_mean", None)
|
||||
latents_std = getattr(video_vae.config, "latents_std", None)
|
||||
if latents_mean is None or latents_std is None:
|
||||
return latents
|
||||
mean = torch.tensor(
|
||||
latents_mean, device=latents.device, dtype=latents.dtype
|
||||
).view(1, video_vae.config.z_dim, 1, 1, 1)
|
||||
inv_std = (
|
||||
1.0 / torch.tensor(latents_std, device=latents.device, dtype=latents.dtype)
|
||||
).view(1, video_vae.config.z_dim, 1, 1, 1)
|
||||
return (latents - mean) * inv_std
|
||||
|
||||
def denormalize_video_latents(
|
||||
self, latents: torch.Tensor, video_vae
|
||||
) -> torch.Tensor:
|
||||
|
||||
@@ -12,11 +12,15 @@ from sglang.multimodal_gen.configs.models.dits.qwenimage import (
|
||||
QwenImageEditPlus_2511_DitConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.models.encoders.qwen_image import Qwen2_5VLConfig
|
||||
from sglang.multimodal_gen.configs.models.vaes.base import (
|
||||
get_channelwise_decode_scale_and_shift,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.models.vaes.qwenimage import QwenImageVAEConfig
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.base import (
|
||||
ImagePipelineConfig,
|
||||
ModelTaskType,
|
||||
maybe_unpad_latents,
|
||||
pack_latents_2x2,
|
||||
pad_text_embeddings_with_mask,
|
||||
shard_rotary_emb_for_sp,
|
||||
)
|
||||
@@ -141,19 +145,6 @@ def _shard_qwen_edit_freqs_cis_for_sp(freqs_cis, noisy_img_seq_len, device):
|
||||
)
|
||||
|
||||
|
||||
# Copied from diffusers.pipelines.qwenimage.pipeline_qwenimage.QwenImagePipeline._pack_latents
|
||||
def _pack_latents(latents, batch_size, num_channels_latents, height, width):
|
||||
latents = latents.view(
|
||||
batch_size, num_channels_latents, height // 2, 2, width // 2, 2
|
||||
)
|
||||
latents = latents.permute(0, 2, 4, 1, 3, 5)
|
||||
latents = latents.reshape(
|
||||
batch_size, (height // 2) * (width // 2), num_channels_latents * 4
|
||||
)
|
||||
|
||||
return latents
|
||||
|
||||
|
||||
@dataclass
|
||||
class QwenImagePipelineConfig(QwenImageRolloutPipelineMixin, ImagePipelineConfig):
|
||||
"""Configuration for the QwenImage pipeline."""
|
||||
@@ -292,19 +283,14 @@ class QwenImagePipelineConfig(QwenImageRolloutPipelineMixin, ImagePipelineConfig
|
||||
width = 2 * (batch.width // (vae_scale_factor * 2))
|
||||
num_channels_latents = self.dit_config.arch_config.in_channels // 4
|
||||
# pack latents
|
||||
return _pack_latents(latents, batch_size, num_channels_latents, height, width)
|
||||
return pack_latents_2x2(
|
||||
latents, batch_size, num_channels_latents, height, width
|
||||
)
|
||||
|
||||
def get_decode_scale_and_shift(self, device, dtype, vae):
|
||||
vae_arch_config = self.vae_config.arch_config
|
||||
scaling_factor = 1.0 / torch.tensor(
|
||||
vae_arch_config.latents_std, device=device
|
||||
).view(1, vae_arch_config.z_dim, 1, 1, 1).to(device, dtype)
|
||||
shift_factor = (
|
||||
torch.tensor(vae_arch_config.latents_mean)
|
||||
.view(1, vae_arch_config.z_dim, 1, 1, 1)
|
||||
.to(device, dtype)
|
||||
return get_channelwise_decode_scale_and_shift(
|
||||
self.vae_config.arch_config, device, dtype
|
||||
)
|
||||
return scaling_factor, shift_factor
|
||||
|
||||
@staticmethod
|
||||
def get_freqs_cis(img_shapes, txt_seq_lens, rotary_emb, device, dtype):
|
||||
@@ -594,7 +580,7 @@ class QwenImageEditPipelineConfig(QwenImagePipelineConfig):
|
||||
image_latents = latent_condition
|
||||
image_latent_height, image_latent_width = image_latents.shape[3:]
|
||||
num_channels_latents = self.dit_config.arch_config.in_channels // 4
|
||||
image_latents = _pack_latents(
|
||||
image_latents = pack_latents_2x2(
|
||||
image_latents,
|
||||
batch_size,
|
||||
num_channels_latents,
|
||||
@@ -639,21 +625,6 @@ VAE_IMAGE_SIZE = 1024 * 1024
|
||||
class QwenImageEditPlusPipelineConfig(QwenImageEditPipelineConfig):
|
||||
task_type: ModelTaskType = ModelTaskType.I2I
|
||||
|
||||
def _get_condition_image_sizes(self, batch) -> list[tuple[int, int]]:
|
||||
image = batch.condition_image
|
||||
if not isinstance(image, list):
|
||||
image = [image]
|
||||
|
||||
condition_image_sizes = []
|
||||
for img in image:
|
||||
image_width, image_height = img.size
|
||||
edit_width, edit_height, _ = _calculate_dimensions(
|
||||
VAE_IMAGE_SIZE, image_width / image_height
|
||||
)
|
||||
condition_image_sizes.append((edit_width, edit_height))
|
||||
|
||||
return condition_image_sizes
|
||||
|
||||
def prepare_image_processor_kwargs(self, batch, neg=False) -> dict:
|
||||
prompt = batch.prompt if not neg else batch.negative_prompt
|
||||
if not prompt:
|
||||
@@ -686,14 +657,6 @@ class QwenImageEditPlusPipelineConfig(QwenImageEditPipelineConfig):
|
||||
def prepare_calculated_size(self, image):
|
||||
return self.calculate_vae_image_size(image, image.width, image.height)
|
||||
|
||||
def resize_condition_image(self, images, target_width, target_height):
|
||||
if not isinstance(images, list):
|
||||
images = [images]
|
||||
new_images = []
|
||||
for img, width, height in zip(images, target_width, target_height):
|
||||
new_images.append(resize(img, height, width, resize_mode="default"))
|
||||
return new_images
|
||||
|
||||
def calculate_condition_image_size(self, image, width, height) -> tuple[int, int]:
|
||||
calculated_width, calculated_height, _ = _calculate_dimensions(
|
||||
CONDITION_IMAGE_SIZE, width / height
|
||||
|
||||
@@ -11,7 +11,10 @@ from sglang.multimodal_gen.configs.models.dits.sana_wm import SanaWMConfig
|
||||
from sglang.multimodal_gen.configs.models.encoders import BaseEncoderOutput
|
||||
from sglang.multimodal_gen.configs.models.encoders.base import EncoderConfig
|
||||
from sglang.multimodal_gen.configs.models.encoders.gemma2 import Gemma2Config
|
||||
from sglang.multimodal_gen.configs.models.vaes.ltx_video import LTXVideoVAEConfig
|
||||
from sglang.multimodal_gen.configs.models.vaes.ltx_video import (
|
||||
LTXVideoVAEConfig,
|
||||
get_ltx_video_decode_scale_and_shift,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.base import (
|
||||
ModelTaskType,
|
||||
PipelineConfig,
|
||||
@@ -286,42 +289,9 @@ class SanaWMPipelineConfig(PipelineConfig):
|
||||
return latents
|
||||
|
||||
def get_decode_scale_and_shift(self, device, dtype, vae):
|
||||
"""Invert the LTX-2 latent normalization used before denoising.
|
||||
|
||||
SANA-WM uses the LTX-2 VAE. Upstream encodes as
|
||||
``(z - latents_mean) * scaling_factor / latents_std`` and decodes by
|
||||
applying the inverse transform.
|
||||
"""
|
||||
latents_mean = getattr(vae, "latents_mean", None)
|
||||
latents_std = getattr(vae, "latents_std", None)
|
||||
|
||||
scaling_factor = (
|
||||
getattr(getattr(vae, "config", None), "scaling_factor", None)
|
||||
or getattr(vae, "scaling_factor", None)
|
||||
or getattr(self.vae_config.arch_config, "scaling_factor", None)
|
||||
or 1.0
|
||||
return get_ltx_video_decode_scale_and_shift(
|
||||
device, dtype, vae, self.vae_config.arch_config
|
||||
)
|
||||
if isinstance(scaling_factor, (int, float)) and float(scaling_factor) == 0.0:
|
||||
scaling_factor = 1.0
|
||||
|
||||
if isinstance(latents_mean, torch.Tensor) and isinstance(
|
||||
latents_std, torch.Tensor
|
||||
):
|
||||
latents_mean = latents_mean.to(device=device, dtype=dtype).view(
|
||||
1, -1, 1, 1, 1
|
||||
)
|
||||
latents_std = latents_std.to(device=device, dtype=dtype).view(
|
||||
1, -1, 1, 1, 1
|
||||
)
|
||||
sf = torch.tensor(float(scaling_factor), device=device, dtype=dtype).view(
|
||||
1, 1, 1, 1, 1
|
||||
)
|
||||
return sf / latents_std, latents_mean
|
||||
|
||||
sf = torch.tensor(float(scaling_factor), device=device, dtype=dtype).view(
|
||||
1, 1, 1, 1, 1
|
||||
)
|
||||
return sf, None
|
||||
|
||||
|
||||
class SanaWMRealtimeConfig(SanaWMPipelineConfig):
|
||||
|
||||
@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Any
|
||||
|
||||
from sglang.multimodal_gen.configs.sample.sampling_params import (
|
||||
DataType,
|
||||
SamplingParams,
|
||||
_sanitize_filename,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.task_type import ModelTaskType
|
||||
@@ -65,15 +66,7 @@ class ActionSamplingParams:
|
||||
if env_steps is not None and self.num_inference_steps is not None:
|
||||
self.num_inference_steps = int(env_steps)
|
||||
|
||||
def build_request_extra(self) -> dict[str, Any]:
|
||||
extra = {}
|
||||
diffusers_kwargs = getattr(self, "diffusers_kwargs", None)
|
||||
if diffusers_kwargs:
|
||||
extra["diffusers_kwargs"] = diffusers_kwargs
|
||||
explicit_fields = getattr(self, "_explicit_fields", None)
|
||||
if explicit_fields is not None:
|
||||
extra["explicit_fields"] = sorted(explicit_fields)
|
||||
return extra
|
||||
build_request_extra = SamplingParams.build_request_extra
|
||||
|
||||
def apply_request_extra(self, req: Any) -> None:
|
||||
req.extra.update(self.build_request_extra())
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
import math
|
||||
import os
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field
|
||||
@@ -8,6 +7,7 @@ from typing import Any, ClassVar
|
||||
import msgspec
|
||||
|
||||
from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
|
||||
from sglang.multimodal_gen.configs.utils import optional_positive_finite_float
|
||||
|
||||
_MINIMAX_H3_MAX_SIGNED_SEED = (1 << 63) - 1
|
||||
|
||||
@@ -23,17 +23,6 @@ def _optional_unit_float(value: Any, field_name: str) -> float | None:
|
||||
return out
|
||||
|
||||
|
||||
def _optional_positive_finite_float(value: Any, field_name: str) -> float | None:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
||||
raise ValueError(f"{field_name} must be a number")
|
||||
out = float(value)
|
||||
if not math.isfinite(out) or out <= 0.0:
|
||||
raise ValueError(f"{field_name} must be a positive finite number")
|
||||
return out
|
||||
|
||||
|
||||
@dataclass
|
||||
class MiniMaxH3SamplingParams(SamplingParams):
|
||||
# The video/audio sigma schedules include both interval endpoints.
|
||||
@@ -226,8 +215,8 @@ class MiniMaxH3SamplingParams(SamplingParams):
|
||||
if field_name in self.target
|
||||
}
|
||||
super()._validate()
|
||||
_optional_positive_finite_float(self.flow_shift, "flow_shift")
|
||||
_optional_positive_finite_float(self.audio_flow_shift, "audio_flow_shift")
|
||||
optional_positive_finite_float(self.flow_shift, "flow_shift")
|
||||
optional_positive_finite_float(self.audio_flow_shift, "audio_flow_shift")
|
||||
if self.enable_frame_interpolation:
|
||||
raise ValueError(
|
||||
"MiniMax H3 does not support enable_frame_interpolation: the "
|
||||
|
||||
@@ -758,10 +758,6 @@ class SamplingParams:
|
||||
|
||||
RLRolloutArgs.validate_sampling_params(self)
|
||||
|
||||
def check_sampling_param(self):
|
||||
# Keep backward-compatibility for old call sites.
|
||||
self._validate()
|
||||
|
||||
def _validate_with_pipeline_config(self, pipeline_config):
|
||||
"""
|
||||
check if the sampling params is compatible and valid with server_args
|
||||
|
||||
@@ -1,12 +1,23 @@
|
||||
# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo
|
||||
|
||||
import argparse
|
||||
import math
|
||||
import os
|
||||
from dataclasses import fields
|
||||
from operator import attrgetter
|
||||
from typing import Any
|
||||
|
||||
|
||||
def optional_positive_finite_float(value: Any, field_name: str) -> float | None:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
||||
raise ValueError(f"{field_name} must be a number")
|
||||
out = float(value)
|
||||
if not math.isfinite(out) or out <= 0.0:
|
||||
raise ValueError(f"{field_name} must be a positive finite number")
|
||||
return out
|
||||
|
||||
|
||||
def expand_path_fields(obj) -> None:
|
||||
"""Expand paths in dataclass configuration without modifying other fields."""
|
||||
for f in fields(obj):
|
||||
@@ -81,15 +92,3 @@ def update_config_from_args(
|
||||
args_dict.pop(key)
|
||||
|
||||
return len(args_to_remove) > 0
|
||||
|
||||
|
||||
def clean_cli_args(args: argparse.Namespace) -> dict[str, Any]:
|
||||
"""
|
||||
Clean the arguments by removing the ones that not explicitly provided by the user.
|
||||
"""
|
||||
provided_args = {}
|
||||
for k, v in vars(args).items():
|
||||
if v is not None and hasattr(args, "_provided") and k in args._provided:
|
||||
provided_args[k] = v
|
||||
|
||||
return provided_args
|
||||
|
||||
@@ -5,12 +5,21 @@
|
||||
|
||||
import logging
|
||||
import os
|
||||
import warnings
|
||||
from typing import TYPE_CHECKING, Any, Callable
|
||||
|
||||
from sglang.multimodal_gen.runtime.utils.common import get_bool_env_var
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
if "SGLANG_DIFFUSION_TRACE_FUNCTION" in os.environ:
|
||||
warnings.warn(
|
||||
"SGLANG_DIFFUSION_TRACE_FUNCTION was unused and has been removed. "
|
||||
"Use Python profiling tools for function-level tracing.",
|
||||
FutureWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
SGLANG_DIFFUSION_NCCL_SO_PATH: str | None = None
|
||||
LD_LIBRARY_PATH: str | None = None
|
||||
@@ -20,7 +29,6 @@ if TYPE_CHECKING:
|
||||
SGLANG_DIFFUSION_CONFIG_ROOT: str = os.path.expanduser("~/.config/sgl_diffusion")
|
||||
SGLANG_DIFFUSION_LOGGING_LEVEL: str = "INFO"
|
||||
SGLANG_DIFFUSION_LOGGING_PREFIX: str = ""
|
||||
SGLANG_DIFFUSION_TRACE_FUNCTION: int = 0
|
||||
SGLANG_DIFFUSION_DISABLE_EARLY_VAE_DECODER_CAST: bool = False
|
||||
SGLANG_DIFFUSION_DISABLE_VAE_DECODER_STORE: bool = False
|
||||
SGLANG_DIFFUSION_DISABLE_MAPPED_WILLNEED: bool = False
|
||||
@@ -229,10 +237,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
),
|
||||
# if set, SGLANG_DIFFUSION_LOGGING_PREFIX will be prepended to all log messages
|
||||
"SGLANG_DIFFUSION_LOGGING_PREFIX": _lazy_str("SGLANG_DIFFUSION_LOGGING_PREFIX", ""),
|
||||
# Trace function calls
|
||||
# If set to 1, sgl_diffusion will trace function calls
|
||||
# Useful for debugging
|
||||
"SGLANG_DIFFUSION_TRACE_FUNCTION": _lazy_int("SGLANG_DIFFUSION_TRACE_FUNCTION", 0),
|
||||
# Path to the attention configuration file. Only used for sliding tile
|
||||
# attention for now.
|
||||
"SGLANG_DIFFUSION_ATTENTION_CONFIG": _lazy_path(
|
||||
|
||||
@@ -30,16 +30,8 @@ def is_ideogram_transformer(current_model: Any, call_kwargs: dict) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def _unwrap_model(current_model: Any) -> Any:
|
||||
for attr in ("module", "_orig_mod"):
|
||||
wrapped = getattr(current_model, attr, None)
|
||||
if wrapped is not None:
|
||||
current_model = wrapped
|
||||
return current_model
|
||||
|
||||
|
||||
def _dynamic_mask_meta(current_model: Any) -> DynamicVarlenMaskMeta:
|
||||
model = _unwrap_model(current_model)
|
||||
model = bcg_utils.unwrap_model(current_model)
|
||||
meta = getattr(model, _DYNAMIC_MASK_META_ATTR, None)
|
||||
if not isinstance(meta, DynamicVarlenMaskMeta):
|
||||
meta = DynamicVarlenMaskMeta()
|
||||
|
||||
@@ -55,16 +55,8 @@ def _pad_caption(obj: Any, *, target: int) -> Any:
|
||||
return obj
|
||||
|
||||
|
||||
def _unwrap_model(current_model: Any) -> Any:
|
||||
for attr in ("module", "_orig_mod"):
|
||||
wrapped = getattr(current_model, attr, None)
|
||||
if wrapped is not None:
|
||||
current_model = wrapped
|
||||
return current_model
|
||||
|
||||
|
||||
def _build_caption_freqs(current_model: Any, *, target: int, device: torch.device):
|
||||
rotary_emb = getattr(_unwrap_model(current_model), "rotary_emb", None)
|
||||
rotary_emb = getattr(bcg_utils.unwrap_model(current_model), "rotary_emb", None)
|
||||
if rotary_emb is None:
|
||||
return None
|
||||
|
||||
|
||||
@@ -249,6 +249,14 @@ def pad_masked_prompt_kwargs(call_kwargs: dict, buckets: tuple[int, ...]) -> dic
|
||||
return out
|
||||
|
||||
|
||||
def unwrap_model(current_model: Any) -> Any:
|
||||
for attr in ("module", "_orig_mod"):
|
||||
wrapped = getattr(current_model, attr, None)
|
||||
if wrapped is not None:
|
||||
current_model = wrapped
|
||||
return current_model
|
||||
|
||||
|
||||
def transformer_class_name_matches(current_model: Any, needle: str) -> bool:
|
||||
"""True when ``current_model`` (or its ``module`` / ``_orig_mod`` wrapper)
|
||||
is a transformer whose qualified class name contains ``needle``."""
|
||||
|
||||
@@ -1,28 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Compatibility shim for disaggregated diffusion argument helpers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
|
||||
from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
|
||||
from sglang.multimodal_gen.runtime.server_args.disagg import DisaggServerArgsMixin
|
||||
|
||||
# Keep the historical disagg_args import path working.
|
||||
DISAGG_RESULT_PORT_OFFSETS = DisaggServerArgsMixin.DISAGG_RESULT_PORT_OFFSETS
|
||||
DisaggArgsMixin = DisaggServerArgsMixin
|
||||
|
||||
|
||||
def add_disagg_cli_args(parser: argparse.ArgumentParser) -> None:
|
||||
"""Register disaggregated-diffusion CLI args through ServerArgs."""
|
||||
|
||||
from sglang.multimodal_gen.runtime.server_args import ServerArgs
|
||||
|
||||
ServerArgs.add_disagg_cli_args(parser)
|
||||
|
||||
|
||||
def convert_disagg_role_string(kwargs: dict) -> None:
|
||||
"""Convert ``disagg_role`` from string to ``RoleType`` enum in-place."""
|
||||
|
||||
if "disagg_role" in kwargs and isinstance(kwargs["disagg_role"], str):
|
||||
kwargs["disagg_role"] = RoleType.from_string(kwargs["disagg_role"])
|
||||
@@ -2,11 +2,8 @@
|
||||
"""Dispatch policies for multi-instance disaggregated diffusion pipelines."""
|
||||
|
||||
import abc
|
||||
import logging
|
||||
import threading
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DispatchPolicy(abc.ABC):
|
||||
def __init__(self, num_instances: int):
|
||||
@@ -19,16 +16,9 @@ class DispatchPolicy(abc.ABC):
|
||||
return self._num_instances
|
||||
|
||||
@abc.abstractmethod
|
||||
def select(self, active_counts: list[int] | None = None) -> int: ...
|
||||
|
||||
def select_with_capacity(self, free_slots: list[int]) -> int | None:
|
||||
"""Select an instance that has free capacity, or None if all full."""
|
||||
if not any(s > 0 for s in free_slots):
|
||||
return None
|
||||
return self.select(active_counts=None)
|
||||
|
||||
def record_completion(self, instance_id: int) -> None:
|
||||
pass
|
||||
...
|
||||
|
||||
|
||||
class RoundRobin(DispatchPolicy):
|
||||
@@ -37,12 +27,6 @@ class RoundRobin(DispatchPolicy):
|
||||
self._lock = threading.Lock()
|
||||
self._next = 0
|
||||
|
||||
def select(self, active_counts: list[int] | None = None) -> int:
|
||||
with self._lock:
|
||||
chosen = self._next
|
||||
self._next = (self._next + 1) % self._num_instances
|
||||
return chosen
|
||||
|
||||
def select_with_capacity(self, free_slots: list[int]) -> int | None:
|
||||
with self._lock:
|
||||
for _ in range(self._num_instances):
|
||||
@@ -56,43 +40,11 @@ class RoundRobin(DispatchPolicy):
|
||||
class MaxFreeSlotsFirst(DispatchPolicy):
|
||||
"""Dispatch to the instance with the most free slots."""
|
||||
|
||||
def __init__(self, num_instances: int, max_slots_per_instance: int = 1):
|
||||
def __init__(self, num_instances: int):
|
||||
super().__init__(num_instances)
|
||||
self._max_slots = max_slots_per_instance
|
||||
self._lock = threading.Lock()
|
||||
self._tiebreak = 0
|
||||
|
||||
def select(self, active_counts: list[int] | None = None) -> int:
|
||||
with self._lock:
|
||||
if active_counts is None or len(active_counts) != self._num_instances:
|
||||
chosen = self._tiebreak % self._num_instances
|
||||
self._tiebreak += 1
|
||||
return chosen
|
||||
|
||||
best_id = 0
|
||||
best_free = self._max_slots - active_counts[0]
|
||||
for i in range(1, self._num_instances):
|
||||
free = self._max_slots - active_counts[i]
|
||||
if free > best_free:
|
||||
best_free = free
|
||||
best_id = i
|
||||
elif free == best_free:
|
||||
if i == (self._tiebreak % self._num_instances):
|
||||
best_id = i
|
||||
|
||||
self._tiebreak += 1
|
||||
|
||||
if best_free <= 0:
|
||||
logger.warning(
|
||||
"All %d instances are at capacity (%d slots each), "
|
||||
"dispatching to instance %d anyway",
|
||||
self._num_instances,
|
||||
self._max_slots,
|
||||
best_id,
|
||||
)
|
||||
|
||||
return best_id
|
||||
|
||||
def select_with_capacity(self, free_slots: list[int]) -> int | None:
|
||||
with self._lock:
|
||||
best_id = -1
|
||||
@@ -121,26 +73,10 @@ class PoolDispatcher:
|
||||
num_denoisers: int,
|
||||
num_decoders: int,
|
||||
policy_name: str = "round_robin",
|
||||
**kwargs,
|
||||
):
|
||||
self.encoder_policy = create_dispatch_policy(
|
||||
policy_name, num_encoders, **kwargs
|
||||
)
|
||||
self.denoiser_policy = create_dispatch_policy(
|
||||
policy_name, num_denoisers, **kwargs
|
||||
)
|
||||
self.decoder_policy = create_dispatch_policy(
|
||||
policy_name, num_decoders, **kwargs
|
||||
)
|
||||
|
||||
def select_encoder(self, active_counts: list[int] | None = None) -> int:
|
||||
return self.encoder_policy.select(active_counts)
|
||||
|
||||
def select_denoiser(self, active_counts: list[int] | None = None) -> int:
|
||||
return self.denoiser_policy.select(active_counts)
|
||||
|
||||
def select_decoder(self, active_counts: list[int] | None = None) -> int:
|
||||
return self.decoder_policy.select(active_counts)
|
||||
self.encoder_policy = create_dispatch_policy(policy_name, num_encoders)
|
||||
self.denoiser_policy = create_dispatch_policy(policy_name, num_denoisers)
|
||||
self.decoder_policy = create_dispatch_policy(policy_name, num_decoders)
|
||||
|
||||
def select_encoder_with_capacity(self, free_slots: list[int]) -> int | None:
|
||||
return self.encoder_policy.select_with_capacity(free_slots)
|
||||
@@ -152,7 +88,7 @@ class PoolDispatcher:
|
||||
return self.decoder_policy.select_with_capacity(free_slots)
|
||||
|
||||
|
||||
def create_dispatch_policy(name: str, num_instances: int, **kwargs) -> DispatchPolicy:
|
||||
def create_dispatch_policy(name: str, num_instances: int) -> DispatchPolicy:
|
||||
policies = {
|
||||
"round_robin": RoundRobin,
|
||||
"max_free_slots": MaxFreeSlotsFirst,
|
||||
@@ -162,4 +98,4 @@ def create_dispatch_policy(name: str, num_instances: int, **kwargs) -> DispatchP
|
||||
raise ValueError(
|
||||
f"Unknown dispatch policy '{name}'. Available: {list(policies.keys())}"
|
||||
)
|
||||
return cls(num_instances=num_instances, **kwargs)
|
||||
return cls(num_instances=num_instances)
|
||||
|
||||
@@ -6,12 +6,6 @@ import time
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass
|
||||
class _RequestTiming:
|
||||
start_time: float
|
||||
stage_start: float = 0.0
|
||||
|
||||
|
||||
@dataclass
|
||||
class RoleStats:
|
||||
role: str
|
||||
@@ -54,7 +48,7 @@ class DisaggMetrics:
|
||||
self._failed = 0
|
||||
self._timed_out = 0
|
||||
|
||||
self._in_flight: dict[str, _RequestTiming] = {}
|
||||
self._in_flight: dict[str, float] = {}
|
||||
|
||||
self._last_latency = 0.0
|
||||
self._max_latency = 0.0
|
||||
@@ -71,14 +65,14 @@ class DisaggMetrics:
|
||||
|
||||
def record_request_start(self, request_id: str) -> None:
|
||||
with self._lock:
|
||||
self._in_flight[request_id] = _RequestTiming(start_time=time.monotonic())
|
||||
self._in_flight[request_id] = time.monotonic()
|
||||
|
||||
def record_request_complete(self, request_id: str) -> None:
|
||||
now = time.monotonic()
|
||||
with self._lock:
|
||||
timing = self._in_flight.pop(request_id, None)
|
||||
if timing is not None:
|
||||
latency = now - timing.start_time
|
||||
start_time = self._in_flight.pop(request_id, None)
|
||||
if start_time is not None:
|
||||
latency = now - start_time
|
||||
self._last_latency = latency
|
||||
self._max_latency = max(self._max_latency, latency)
|
||||
self._total_latency += latency
|
||||
|
||||
@@ -865,8 +865,8 @@ class DiffusionServer:
|
||||
if idx is None:
|
||||
break
|
||||
entry = self._denoiser_tta.popleft()
|
||||
self._transfer_dispatch_to_denoiser(
|
||||
entry.request_id, entry.transfer_state, idx
|
||||
self._transfer_dispatch_to_receiver(
|
||||
entry.request_id, entry.transfer_state, idx, RoleType.DENOISER
|
||||
)
|
||||
|
||||
def _drain_decoder_tta(self) -> None:
|
||||
@@ -877,8 +877,8 @@ class DiffusionServer:
|
||||
if idx is None:
|
||||
break
|
||||
entry = self._decoder_tta.popleft()
|
||||
self._transfer_dispatch_to_decoder(
|
||||
entry.request_id, entry.transfer_state, idx
|
||||
self._transfer_dispatch_to_receiver(
|
||||
entry.request_id, entry.transfer_state, idx, RoleType.DECODER
|
||||
)
|
||||
|
||||
def _extract_request_id(self, frames: list) -> str | None:
|
||||
@@ -1161,36 +1161,57 @@ class DiffusionServer:
|
||||
)
|
||||
receiver_pushes[receiver_idx].send_multipart(encode_transfer_msg(alloc_msg))
|
||||
|
||||
def _transfer_dispatch_to_denoiser(
|
||||
self, request_id: str, p2p: _TransferRequestState, denoiser_idx: int
|
||||
def _transfer_dispatch_to_receiver(
|
||||
self,
|
||||
request_id: str,
|
||||
p2p: _TransferRequestState,
|
||||
receiver_idx: int,
|
||||
role: RoleType,
|
||||
) -> None:
|
||||
self._denoiser_free_slots[denoiser_idx] -= 1
|
||||
p2p.receiver_instance = denoiser_idx
|
||||
if role == RoleType.DENOISER:
|
||||
free_slots = self._denoiser_free_slots
|
||||
peers = self._denoiser_peers
|
||||
sender_pushes = self._encoder_pushes
|
||||
receiver_pushes = self._denoiser_pushes
|
||||
source_role = "encoder"
|
||||
running_state = RequestState.DENOISING_RUNNING
|
||||
elif role == RoleType.DECODER:
|
||||
free_slots = self._decoder_free_slots
|
||||
peers = self._decoder_peers
|
||||
sender_pushes = self._denoiser_pushes
|
||||
receiver_pushes = self._decoder_pushes
|
||||
source_role = "denoiser"
|
||||
running_state = RequestState.DECODER_RUNNING
|
||||
else:
|
||||
raise ValueError(f"Invalid transfer receiver role: {role}")
|
||||
|
||||
free_slots[receiver_idx] -= 1
|
||||
p2p.receiver_instance = receiver_idx
|
||||
|
||||
try:
|
||||
self._tracker.transition(
|
||||
request_id,
|
||||
RequestState.DENOISING_RUNNING,
|
||||
denoiser_instance=denoiser_idx,
|
||||
running_state,
|
||||
**{f"{role.value}_instance": receiver_idx},
|
||||
)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
peer_info = self._denoiser_peers.get(denoiser_idx, {})
|
||||
peer_info = peers.get(receiver_idx, {})
|
||||
if not self._try_fast_path_push(
|
||||
request_id=request_id,
|
||||
p2p=p2p,
|
||||
receiver_peer_info=peer_info,
|
||||
sender_pushes=self._encoder_pushes,
|
||||
receiver_role_label="denoiser",
|
||||
receiver_idx=denoiser_idx,
|
||||
sender_pushes=sender_pushes,
|
||||
receiver_role_label=role.value,
|
||||
receiver_idx=receiver_idx,
|
||||
):
|
||||
self._send_slow_path_alloc(
|
||||
request_id=request_id,
|
||||
p2p=p2p,
|
||||
receiver_pushes=self._denoiser_pushes,
|
||||
receiver_idx=denoiser_idx,
|
||||
source_role="encoder",
|
||||
receiver_pushes=receiver_pushes,
|
||||
receiver_idx=receiver_idx,
|
||||
source_role=source_role,
|
||||
)
|
||||
|
||||
def _handle_transfer_allocated(self, msg: dict) -> None:
|
||||
@@ -1378,38 +1399,6 @@ class DiffusionServer:
|
||||
|
||||
self._transfer_state.pop(request_id, None)
|
||||
|
||||
def _transfer_dispatch_to_decoder(
|
||||
self, request_id: str, p2p: _TransferRequestState, decoder_idx: int
|
||||
) -> None:
|
||||
self._decoder_free_slots[decoder_idx] -= 1
|
||||
p2p.receiver_instance = decoder_idx
|
||||
|
||||
try:
|
||||
self._tracker.transition(
|
||||
request_id,
|
||||
RequestState.DECODER_RUNNING,
|
||||
decoder_instance=decoder_idx,
|
||||
)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
peer_info = self._decoder_peers.get(decoder_idx, {})
|
||||
if not self._try_fast_path_push(
|
||||
request_id=request_id,
|
||||
p2p=p2p,
|
||||
receiver_peer_info=peer_info,
|
||||
sender_pushes=self._denoiser_pushes,
|
||||
receiver_role_label="decoder",
|
||||
receiver_idx=decoder_idx,
|
||||
):
|
||||
self._send_slow_path_alloc(
|
||||
request_id=request_id,
|
||||
p2p=p2p,
|
||||
receiver_pushes=self._decoder_pushes,
|
||||
receiver_idx=decoder_idx,
|
||||
source_role="denoiser",
|
||||
)
|
||||
|
||||
def _transfer_return_to_client_from_msg(self, request_id: str, msg: dict) -> None:
|
||||
with self._lock:
|
||||
client_identity = self._pending.pop(request_id, None)
|
||||
|
||||
@@ -427,9 +427,7 @@ class SchedulerDisaggMixin:
|
||||
from sglang.multimodal_gen.runtime.disaggregation.metrics import DisaggMetrics
|
||||
|
||||
self._disagg_role = server_args.disagg_role
|
||||
self._disagg_timeout_s = float(getattr(server_args, "disagg_timeout", 600))
|
||||
self._disagg_metrics = None
|
||||
self._disagg_mode = getattr(server_args, "disagg_mode", False)
|
||||
self._pool_work_pull = None
|
||||
self._pool_result_push = None
|
||||
self._transfer_manager = None
|
||||
|
||||
@@ -45,11 +45,6 @@ def sequence_model_parallel_all_gather(
|
||||
return get_sp_group().all_gather(input_, dim)
|
||||
|
||||
|
||||
def sequence_model_parallel_all_reduce(input_: torch.Tensor) -> torch.Tensor:
|
||||
"""All-reduce the input tensor across model parallel group."""
|
||||
return get_sp_group().all_reduce(input_)
|
||||
|
||||
|
||||
def cfg_model_parallel_all_gather(
|
||||
input_: torch.Tensor, dim: int = -1, separate_tensors: bool = False
|
||||
) -> torch.Tensor:
|
||||
|
||||
-1
@@ -270,7 +270,6 @@ class DeviceCommunicatorBase:
|
||||
self.world_size = dist.get_world_size(cpu_group)
|
||||
self.ranks = dist.get_process_group_ranks(cpu_group)
|
||||
self.global_rank = dist.get_rank()
|
||||
self.global_world_size = dist.get_world_size()
|
||||
self.rank_in_group = dist.get_group_rank(self.cpu_group, self.global_rank)
|
||||
|
||||
def all_reduce(
|
||||
|
||||
@@ -169,7 +169,6 @@ class GroupCoordinator:
|
||||
rank_in_group: int # rank inside the group
|
||||
cpu_group: ProcessGroup # group for CPU communication
|
||||
device_group: ProcessGroup # group for device communication
|
||||
use_device_communicator: bool # whether to use device communicator
|
||||
device_communicator: DeviceCommunicatorBase # all_to_all_4D communicator
|
||||
|
||||
def __init__(
|
||||
@@ -207,7 +206,6 @@ class GroupCoordinator:
|
||||
# TODO: fix it for other platforms
|
||||
self.device = get_local_torch_device()
|
||||
|
||||
self.use_device_communicator = use_device_communicator
|
||||
self.device_communicator: DeviceCommunicatorBase = None # type: ignore
|
||||
if use_device_communicator and self.world_size > 1:
|
||||
communicator_cls = _resolve_all_to_all_communicator_cls()
|
||||
@@ -228,10 +226,6 @@ class GroupCoordinator:
|
||||
# srt owns topology, dtype, contiguity, and size dispatch for custom ar
|
||||
self._init_srt_custom_allreduce()
|
||||
|
||||
# TODO(will): check if this is needed
|
||||
# self.use_custom_op_call = current_platform.is_cuda_alike()
|
||||
self.use_custom_op_call = False
|
||||
|
||||
def _init_srt_custom_allreduce(self) -> None:
|
||||
custom_allreduce_kwargs = {
|
||||
"group": self.cpu_group,
|
||||
|
||||
@@ -767,11 +767,6 @@ def get_sequence_parallel_world_size() -> int:
|
||||
return get_sp_world_size()
|
||||
|
||||
|
||||
def get_sequence_parallel_rank() -> int:
|
||||
"""Return my rank for the sequence parallel group."""
|
||||
return get_sp_parallel_rank()
|
||||
|
||||
|
||||
def get_ulysses_parallel_world_size() -> int:
|
||||
return get_sp_group().ulysses_world_size
|
||||
|
||||
@@ -803,32 +798,6 @@ def get_ring_ctx() -> tuple[int, int]:
|
||||
return get_ring_parallel_world_size(), get_ring_parallel_rank()
|
||||
|
||||
|
||||
# PP
|
||||
def get_pp_group() -> GroupCoordinator:
|
||||
assert _PP is not None, "pipeline model parallel group is not initialized"
|
||||
return _PP
|
||||
|
||||
|
||||
def get_pipeline_parallel_world_size() -> int:
|
||||
"""Return world size for the pipeline model parallel group."""
|
||||
return get_pp_group().world_size
|
||||
|
||||
|
||||
def get_pipeline_parallel_rank() -> int:
|
||||
"""Return my rank for the pipeline model parallel group."""
|
||||
return get_pp_group().rank_in_group
|
||||
|
||||
|
||||
def is_pipeline_first_stage() -> bool:
|
||||
"""Return True if in the first pipeline model parallel stage, False otherwise."""
|
||||
return get_pipeline_parallel_rank() == 0
|
||||
|
||||
|
||||
def is_pipeline_last_stage() -> bool:
|
||||
"""Return True if in the last pipeline model parallel stage, False otherwise."""
|
||||
return get_pipeline_parallel_rank() == (get_pipeline_parallel_world_size() - 1)
|
||||
|
||||
|
||||
# CFG
|
||||
def get_cfg_group() -> GroupCoordinator:
|
||||
assert _CFG is not None, (
|
||||
@@ -852,47 +821,6 @@ def get_data_parallel_world_size() -> int:
|
||||
return get_dp_world_size()
|
||||
|
||||
|
||||
def get_data_parallel_rank() -> int:
|
||||
"""Return my rank for the data parallel group."""
|
||||
return get_dp_rank()
|
||||
|
||||
|
||||
def is_dp_last_group() -> bool:
|
||||
"""Return True if in the last data parallel group, False otherwise."""
|
||||
return (
|
||||
get_sequence_parallel_rank() == (get_sequence_parallel_world_size() - 1)
|
||||
and get_classifier_free_guidance_rank()
|
||||
== (get_classifier_free_guidance_world_size() - 1)
|
||||
and get_pipeline_parallel_rank() == (get_pipeline_parallel_world_size() - 1)
|
||||
)
|
||||
|
||||
|
||||
def get_dit_world_size() -> int:
|
||||
"""Return world size for the DiT model (excluding VAE)."""
|
||||
return (
|
||||
get_data_parallel_world_size()
|
||||
* get_classifier_free_guidance_world_size()
|
||||
* get_sequence_parallel_world_size()
|
||||
* get_pipeline_parallel_world_size()
|
||||
* get_tensor_model_parallel_world_size()
|
||||
)
|
||||
|
||||
|
||||
def get_vae_parallel_group() -> ProcessGroup:
|
||||
assert _VAE is not None, "VAE parallel group is not initialized"
|
||||
return _VAE
|
||||
|
||||
|
||||
def get_vae_parallel_world_size() -> int:
|
||||
"""Return world size for the VAE parallel group."""
|
||||
return torch.distributed.get_world_size(group=get_vae_parallel_group())
|
||||
|
||||
|
||||
def get_vae_parallel_rank() -> int:
|
||||
"""Return my rank for the VAE parallel group."""
|
||||
return torch.distributed.get_rank(group=get_vae_parallel_group())
|
||||
|
||||
|
||||
def get_decode_parallel_group_coordinator() -> GroupCoordinator:
|
||||
assert _VAE_DECODE is not None, "VAE decode parallel group is not initialized"
|
||||
return _VAE_DECODE
|
||||
|
||||
@@ -215,16 +215,6 @@ class GenerateSubcommand(CLISubcommand):
|
||||
def __init__(self) -> None:
|
||||
self.name = "generate"
|
||||
super().__init__()
|
||||
self.init_arg_names = self._get_init_arg_names()
|
||||
self.generation_arg_names = self._get_generation_arg_names()
|
||||
|
||||
def _get_init_arg_names(self) -> list[str]:
|
||||
"""Get names of arguments for DiffGenerator initialization"""
|
||||
return ["num_gpus", "tp_size", "sp_size", "model_path"]
|
||||
|
||||
def _get_generation_arg_names(self) -> list[str]:
|
||||
"""Get names of arguments for generate_video method"""
|
||||
return [field.name for field in dataclasses.fields(SamplingParams)]
|
||||
|
||||
def cmd(
|
||||
self, args: argparse.Namespace, unknown_args: list[str] | None = None
|
||||
|
||||
@@ -173,10 +173,6 @@ class VideoListResponse(BaseModel):
|
||||
object: str = "list"
|
||||
|
||||
|
||||
class VideoRemixRequest(BaseModel):
|
||||
prompt: str
|
||||
|
||||
|
||||
class RealtimeVideoGenerationsRequest(VideoGenerationsRequest):
|
||||
enhance_prompt: Literal[False] = False
|
||||
type: Literal["init"]
|
||||
|
||||
-1
@@ -38,7 +38,6 @@ if TYPE_CHECKING:
|
||||
|
||||
|
||||
SANA_WM_DEFAULT_SIZE = "1280x704"
|
||||
SANA_WM_DEFAULT_NUM_FRAMES = 1081
|
||||
SANA_WM_DEFAULT_FPS = 16
|
||||
SANA_WM_DEFAULT_STEPS = 4
|
||||
SANA_WM_DEFAULT_GUIDANCE = 1.0
|
||||
|
||||
@@ -41,9 +41,6 @@ class SiluAndMul(CustomOp):
|
||||
return: (num_tokens, d) or (batch_size, seq_len, d)
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
|
||||
def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
|
||||
d = x.shape[-1] // 2
|
||||
output_shape = x.shape[:-1] + (d,)
|
||||
@@ -111,9 +108,6 @@ class GeluAndMul(CustomOp):
|
||||
|
||||
@CustomOp.register("gelu_new")
|
||||
class NewGELU(CustomOp):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
||||
def forward_cuda(self, *args, **kwargs) -> Any:
|
||||
return self.forward_native(*args, **kwargs)
|
||||
|
||||
@@ -129,9 +123,6 @@ class NewGELU(CustomOp):
|
||||
@CustomOp.register("quick_gelu")
|
||||
class QuickGELU(CustomOp):
|
||||
# https://github.com/huggingface/transformers/blob/main/src/transformers/activations.py#L90
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
||||
def forward_cuda(self, *args, **kwargs) -> Any:
|
||||
return self.forward_native(*args, **kwargs)
|
||||
|
||||
|
||||
@@ -196,12 +196,6 @@ class AscendFAMetadata:
|
||||
|
||||
|
||||
class AscendFAMetadataBuilder(AttentionMetadataBuilder):
|
||||
def __init__(self) -> None:
|
||||
pass
|
||||
|
||||
def prepare(self) -> None:
|
||||
pass
|
||||
|
||||
def build(
|
||||
self,
|
||||
**kwargs: dict[str, Any],
|
||||
|
||||
@@ -95,16 +95,6 @@ T = TypeVar("T", bound=AttentionMetadata)
|
||||
class AttentionMetadataBuilder(ABC, Generic[T]):
|
||||
"""Abstract class for attention metadata builders."""
|
||||
|
||||
@abstractmethod
|
||||
def __init__(self) -> None:
|
||||
"""Create the builder, remember some configuration and parameters."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def prepare(self) -> None:
|
||||
"""Prepare for one batch."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def build(
|
||||
self,
|
||||
|
||||
@@ -53,12 +53,6 @@ class BlockSparseAttentionMetadata(AttentionMetadata):
|
||||
|
||||
|
||||
class BlockSparseAttentionMetadataBuilder(AttentionMetadataBuilder):
|
||||
def __init__(self) -> None:
|
||||
pass
|
||||
|
||||
def prepare(self) -> None:
|
||||
pass
|
||||
|
||||
def build(
|
||||
self,
|
||||
current_timestep: int,
|
||||
|
||||
-6
@@ -62,12 +62,6 @@ class CubeSparseAttentionMetadata(AttentionMetadata):
|
||||
|
||||
|
||||
class CubeSparseAttentionMetadataBuilder(AttentionMetadataBuilder):
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def prepare(self):
|
||||
pass
|
||||
|
||||
def build( # type: ignore[override]
|
||||
self,
|
||||
*,
|
||||
|
||||
@@ -322,12 +322,6 @@ class FlashAttentionMetadata:
|
||||
|
||||
|
||||
class FlashAttentionMetadataBuilder(AttentionMetadataBuilder):
|
||||
def __init__(self) -> None:
|
||||
pass
|
||||
|
||||
def prepare(self) -> None:
|
||||
pass
|
||||
|
||||
def build( # type: ignore
|
||||
self,
|
||||
raw_latent_shape=list,
|
||||
@@ -386,7 +380,6 @@ class FlashAttentionImpl(AttentionImpl):
|
||||
self.packed_trailing_padding = extra_impl_args.get(
|
||||
"packed_trailing_padding", False
|
||||
)
|
||||
self.attention_metadata = FlashAttentionMetadata()
|
||||
|
||||
def _request_skip_softmax_threshold(self) -> tuple[bool, float | None]:
|
||||
params = get_request_skip_softmax_params()
|
||||
|
||||
-6
@@ -268,12 +268,6 @@ class HybridWindowAttentionH3Metadata(AttentionMetadata):
|
||||
|
||||
|
||||
class HybridWindowAttentionH3MetadataBuilder(AttentionMetadataBuilder):
|
||||
def __init__(self) -> None:
|
||||
pass
|
||||
|
||||
def prepare(self) -> None:
|
||||
pass
|
||||
|
||||
def build( # type: ignore[override]
|
||||
self,
|
||||
*,
|
||||
|
||||
@@ -54,12 +54,6 @@ class RainFusionAttentionMetadata(AttentionMetadata):
|
||||
|
||||
|
||||
class RainFusionAttentionMetadataBuilder(AttentionMetadataBuilder):
|
||||
def __init__(self) -> None:
|
||||
pass
|
||||
|
||||
def prepare(self) -> None:
|
||||
pass
|
||||
|
||||
def build(
|
||||
self,
|
||||
current_timestep: int,
|
||||
|
||||
@@ -80,12 +80,6 @@ class SlidingTileAttentionMetadata(AttentionMetadata):
|
||||
|
||||
|
||||
class SlidingTileAttentionMetadataBuilder(AttentionMetadataBuilder):
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def prepare(self):
|
||||
pass
|
||||
|
||||
def build( # type: ignore
|
||||
self,
|
||||
STA_param: list[list[Any]],
|
||||
|
||||
+24
-67
@@ -86,12 +86,6 @@ class SparseLinearAttentionMetadata(AttentionMetadata):
|
||||
class SparseLinearAttentionMetadataBuilder(AttentionMetadataBuilder):
|
||||
"""Builder for SparseLinearAttentionMetadata."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
pass
|
||||
|
||||
def prepare(self) -> None:
|
||||
pass
|
||||
|
||||
def build(
|
||||
self,
|
||||
current_timestep: int,
|
||||
@@ -104,6 +98,24 @@ class SparseLinearAttentionMetadataBuilder(AttentionMetadataBuilder):
|
||||
)
|
||||
|
||||
|
||||
def _initialize_linear_attention(model, head_size: int, feature_map: str) -> None:
|
||||
model.proj_l = nn.Linear(head_size, head_size, dtype=torch.float32)
|
||||
model.feature_map_q: Callable[[torch.Tensor], torch.Tensor]
|
||||
model.feature_map_k: Callable[[torch.Tensor], torch.Tensor]
|
||||
if feature_map == "elu":
|
||||
model.feature_map_q = lambda x: F.elu(x) + 1
|
||||
model.feature_map_k = lambda x: F.elu(x) + 1
|
||||
elif feature_map == "relu":
|
||||
model.feature_map_q = F.relu
|
||||
model.feature_map_k = F.relu
|
||||
elif feature_map == "softmax":
|
||||
model.feature_map_q = lambda x: F.softmax(x, dim=-1)
|
||||
model.feature_map_k = lambda x: F.softmax(x, dim=-1)
|
||||
else:
|
||||
raise ValueError(f"Unknown feature map: {feature_map}")
|
||||
model._init_weights()
|
||||
|
||||
|
||||
class SparseLinearAttentionImpl(AttentionImpl, nn.Module):
|
||||
"""Implementation of sparse linear attention for the backend."""
|
||||
|
||||
@@ -131,26 +143,7 @@ class SparseLinearAttentionImpl(AttentionImpl, nn.Module):
|
||||
self.BLKK = BLKK
|
||||
self.dtype = torch.bfloat16 if use_bf16 else torch.float16
|
||||
|
||||
# Learnable linear projection for combining sparse + linear attention
|
||||
self.proj_l = nn.Linear(head_size, head_size, dtype=torch.float32)
|
||||
|
||||
# Feature map for linear attention
|
||||
# Type annotation for callables
|
||||
self.feature_map_q: Callable[[torch.Tensor], torch.Tensor]
|
||||
self.feature_map_k: Callable[[torch.Tensor], torch.Tensor]
|
||||
if feature_map == "elu":
|
||||
self.feature_map_q = lambda x: F.elu(x) + 1
|
||||
self.feature_map_k = lambda x: F.elu(x) + 1
|
||||
elif feature_map == "relu":
|
||||
self.feature_map_q = F.relu
|
||||
self.feature_map_k = F.relu
|
||||
elif feature_map == "softmax":
|
||||
self.feature_map_q = lambda x: F.softmax(x, dim=-1)
|
||||
self.feature_map_k = lambda x: F.softmax(x, dim=-1)
|
||||
else:
|
||||
raise ValueError(f"Unknown feature map: {feature_map}")
|
||||
|
||||
self._init_weights()
|
||||
_initialize_linear_attention(self, head_size, feature_map)
|
||||
|
||||
def _init_weights(self) -> None:
|
||||
"""Initialize projection weights to zero for residual-like behavior."""
|
||||
@@ -322,12 +315,6 @@ class SageSparseLinearAttentionMetadata(AttentionMetadata):
|
||||
class SageSparseLinearAttentionMetadataBuilder(AttentionMetadataBuilder):
|
||||
"""Builder for SageSparseLinearAttentionMetadata."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
pass
|
||||
|
||||
def prepare(self) -> None:
|
||||
pass
|
||||
|
||||
def build(
|
||||
self,
|
||||
current_timestep: int,
|
||||
@@ -369,42 +356,12 @@ class SageSparseLinearAttentionImpl(AttentionImpl, nn.Module):
|
||||
self.topk_ratio = topk_ratio
|
||||
self.dtype = torch.bfloat16 if use_bf16 else torch.float16
|
||||
|
||||
# Learnable linear projection for combining sparse + linear attention
|
||||
self.proj_l = nn.Linear(head_size, head_size, dtype=torch.float32)
|
||||
_initialize_linear_attention(self, head_size, feature_map)
|
||||
|
||||
# Feature map for linear attention
|
||||
# Type annotation for callables
|
||||
self.feature_map_q: Callable[[torch.Tensor], torch.Tensor]
|
||||
self.feature_map_k: Callable[[torch.Tensor], torch.Tensor]
|
||||
if feature_map == "elu":
|
||||
self.feature_map_q = lambda x: F.elu(x) + 1
|
||||
self.feature_map_k = lambda x: F.elu(x) + 1
|
||||
elif feature_map == "relu":
|
||||
self.feature_map_q = F.relu
|
||||
self.feature_map_k = F.relu
|
||||
elif feature_map == "softmax":
|
||||
self.feature_map_q = lambda x: F.softmax(x, dim=-1)
|
||||
self.feature_map_k = lambda x: F.softmax(x, dim=-1)
|
||||
else:
|
||||
raise ValueError(f"Unknown feature map: {feature_map}")
|
||||
|
||||
self._init_weights()
|
||||
|
||||
def _init_weights(self) -> None:
|
||||
"""Initialize projection weights to zero for residual-like behavior."""
|
||||
with torch.no_grad():
|
||||
nn.init.zeros_(self.proj_l.weight)
|
||||
nn.init.zeros_(self.proj_l.bias) # type: ignore[arg-type]
|
||||
|
||||
def _calc_linear_attention_with_torch(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
):
|
||||
kv = torch.matmul(k.transpose(-1, -2), v)
|
||||
k_sum = torch.sum(k, dim=-2, keepdim=True)
|
||||
return torch.matmul(q, kv) / (1e-5 + torch.matmul(q, k_sum.transpose(-1, -2)))
|
||||
_init_weights = SparseLinearAttentionImpl._init_weights
|
||||
_calc_linear_attention_with_torch = (
|
||||
SparseLinearAttentionImpl._calc_linear_attention_with_torch
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
|
||||
-14
@@ -106,21 +106,7 @@ class SparseVideoGen2AttentionMetadata(AttentionMetadata):
|
||||
max_seqlen_k: int | None = None
|
||||
|
||||
|
||||
def _require_kwarg(kwargs: dict[str, Any], name: str) -> Any:
|
||||
if name not in kwargs:
|
||||
raise ValueError(
|
||||
f"Missing required argument for SparseVideoGen2Attention: {name}"
|
||||
)
|
||||
return kwargs[name]
|
||||
|
||||
|
||||
class SparseVideoGen2AttentionMetadataBuilder(AttentionMetadataBuilder):
|
||||
def __init__(self) -> None:
|
||||
pass
|
||||
|
||||
def prepare(self) -> None:
|
||||
pass
|
||||
|
||||
def build( # type: ignore[override]
|
||||
self,
|
||||
current_timestep: int,
|
||||
|
||||
@@ -397,14 +397,6 @@ class SubBlockSparseAttentionMetadata(AttentionMetadata):
|
||||
|
||||
|
||||
class SubBlockSparseAttentionMetadataBuilder(AttentionMetadataBuilder):
|
||||
# The base class declares __init__ abstract, so a builder that does not
|
||||
# override it cannot be instantiated at all.
|
||||
def __init__(self) -> None:
|
||||
pass
|
||||
|
||||
def prepare(self) -> None:
|
||||
pass
|
||||
|
||||
def build( # type: ignore[override]
|
||||
self, current_timestep: int, **kwargs: dict[str, Any]
|
||||
) -> SubBlockSparseAttentionMetadata:
|
||||
|
||||
@@ -174,12 +174,6 @@ def _compute_cur_topk(attn_metadata: VideoSparseAttentionMetadata) -> int:
|
||||
|
||||
|
||||
class VideoSparseAttentionMetadataBuilder(AttentionMetadataBuilder):
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def prepare(self):
|
||||
pass
|
||||
|
||||
def build( # type: ignore
|
||||
self,
|
||||
current_timestep: int,
|
||||
@@ -189,7 +183,6 @@ class VideoSparseAttentionMetadataBuilder(AttentionMetadataBuilder):
|
||||
device: torch.device,
|
||||
**kwargs: dict[str, Any],
|
||||
) -> VideoSparseAttentionMetadata:
|
||||
patch_size = patch_size
|
||||
dit_seq_shape = (
|
||||
raw_latent_shape[0] // patch_size[0],
|
||||
raw_latent_shape[1] // patch_size[1],
|
||||
|
||||
@@ -175,9 +175,6 @@ class VideoSparseAttentionH3MetadataBuilder(AttentionMetadataBuilder):
|
||||
def __init__(self) -> None:
|
||||
self._workspace_cache: dict = {}
|
||||
|
||||
def prepare(self) -> None:
|
||||
pass
|
||||
|
||||
def build( # type: ignore[override]
|
||||
self,
|
||||
current_timestep: int,
|
||||
|
||||
@@ -88,12 +88,6 @@ def pad_input(hidden_states, indices, batch, seqlen):
|
||||
|
||||
|
||||
class VideoMobaAttentionMetadataBuilder(AttentionMetadataBuilder):
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def prepare(self):
|
||||
pass
|
||||
|
||||
def build( # type: ignore
|
||||
self,
|
||||
current_timestep: int,
|
||||
|
||||
@@ -129,14 +129,6 @@ class CustomOp(nn.Module):
|
||||
# since we are not using Inductor, we always return True
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def default_on() -> bool:
|
||||
"""
|
||||
On by default if level < CompilationLevel.PIECEWISE
|
||||
Specifying 'all' or 'none' in custom_op takes precedence.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
# Dictionary of all custom ops (classes, indexed by registered name).
|
||||
# To check if an op with a name is enabled, call .enabled() on the class.
|
||||
# Examples:
|
||||
|
||||
@@ -88,6 +88,27 @@ def adjust_scalar_to_fused_array(
|
||||
return param[shard_id], loaded_weight
|
||||
|
||||
|
||||
def _load_tensor_parallel_weight(
|
||||
param: Parameter,
|
||||
loaded_weight: torch.Tensor,
|
||||
tp_rank: int,
|
||||
shard_dim: int | None,
|
||||
) -> None:
|
||||
is_sharded_weight = getattr(param, "is_sharded_weight", False)
|
||||
param_data = param.data
|
||||
# Already-sharded checkpoints (e.g. bitsandbytes) must not be sliced twice.
|
||||
if shard_dim is not None and not is_sharded_weight:
|
||||
shard_size = param_data.shape[shard_dim]
|
||||
loaded_weight = loaded_weight.narrow(
|
||||
shard_dim, tp_rank * shard_size, shard_size
|
||||
)
|
||||
# AutoFP8 scales can be stored as scalars rather than length-one tensors.
|
||||
if len(loaded_weight.shape) == 0:
|
||||
loaded_weight = loaded_weight.reshape(1)
|
||||
assert param_data.shape == loaded_weight.shape
|
||||
param_data.copy_(loaded_weight)
|
||||
|
||||
|
||||
class LinearMethodBase(QuantizeMethodBase):
|
||||
"""Base class for different (maybe quantized) linear methods."""
|
||||
|
||||
@@ -477,25 +498,9 @@ class ColumnParallelLinear(LinearBase):
|
||||
self.register_parameter("bias", None)
|
||||
|
||||
def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor) -> None:
|
||||
tp_rank = self.tp_rank
|
||||
output_dim = getattr(param, "output_dim", None)
|
||||
|
||||
is_sharded_weight = getattr(param, "is_sharded_weight", False)
|
||||
is_sharded_weight = is_sharded_weight
|
||||
|
||||
param_data = param.data
|
||||
if output_dim is not None and not is_sharded_weight:
|
||||
shard_size = param_data.shape[output_dim]
|
||||
start_idx = tp_rank * shard_size
|
||||
loaded_weight = loaded_weight.narrow(output_dim, start_idx, shard_size)
|
||||
|
||||
# Special case for loading scales off disk, which often do not
|
||||
# have a shape (such as in the case of AutoFP8).
|
||||
if len(loaded_weight.shape) == 0:
|
||||
loaded_weight = loaded_weight.reshape(1)
|
||||
|
||||
assert param_data.shape == loaded_weight.shape
|
||||
param_data.copy_(loaded_weight)
|
||||
_load_tensor_parallel_weight(
|
||||
param, loaded_weight, self.tp_rank, getattr(param, "output_dim", None)
|
||||
)
|
||||
|
||||
def weight_loader_v2(self, param: Parameter, loaded_weight: torch.Tensor) -> None:
|
||||
# Special case for loading scales off disk, which often do not
|
||||
@@ -629,7 +634,6 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
|
||||
is_sharded_weight = getattr(param, "is_sharded_weight", False)
|
||||
# bitsandbytes loads the weights of the specific portion
|
||||
# no need to narrow
|
||||
is_sharded_weight = is_sharded_weight
|
||||
|
||||
param_data = param_data.narrow(output_dim, shard_offset, shard_size)
|
||||
start_idx = tp_rank * shard_size
|
||||
@@ -1027,7 +1031,6 @@ class QKVParallelLinear(ColumnParallelLinear):
|
||||
is_sharded_weight = getattr(param, "is_sharded_weight", False)
|
||||
# bitsandbytes loads the weights of the specific portion
|
||||
# no need to narrow
|
||||
is_sharded_weight = is_sharded_weight
|
||||
|
||||
shard_idx = 0
|
||||
param_data = param_data.narrow(output_dim, shard_offset, shard_size)
|
||||
@@ -1151,26 +1154,9 @@ class RowParallelLinear(LinearBase):
|
||||
self.register_parameter("bias", None)
|
||||
|
||||
def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor):
|
||||
tp_rank = self.tp_rank
|
||||
input_dim = getattr(param, "input_dim", None)
|
||||
is_sharded_weight = getattr(param, "is_sharded_weight", False)
|
||||
# bitsandbytes loads the weights of the specific portion
|
||||
# no need to narrow
|
||||
is_sharded_weight = is_sharded_weight
|
||||
|
||||
param_data = param.data
|
||||
if input_dim is not None and not is_sharded_weight:
|
||||
shard_size = param_data.shape[input_dim]
|
||||
start_idx = tp_rank * shard_size
|
||||
loaded_weight = loaded_weight.narrow(input_dim, start_idx, shard_size)
|
||||
|
||||
# Special case for loading scales off disk, which often do not
|
||||
# have a shape (such as in the case of AutoFP8).
|
||||
if len(loaded_weight.shape) == 0:
|
||||
loaded_weight = loaded_weight.reshape(1)
|
||||
|
||||
assert param_data.shape == loaded_weight.shape
|
||||
param_data.copy_(loaded_weight)
|
||||
_load_tensor_parallel_weight(
|
||||
param, loaded_weight, self.tp_rank, getattr(param, "input_dim", None)
|
||||
)
|
||||
|
||||
def weight_loader_v2(self, param: BasevLLMParameter, loaded_weight: torch.Tensor):
|
||||
# Special case for loading scales off disk, which often do not
|
||||
|
||||
@@ -32,9 +32,6 @@ from sglang.multimodal_gen.runtime.layers.linear import (
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.weight_only_fp8 import (
|
||||
WeightOnlyFP8Linear,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.forward_context import (
|
||||
get_forward_context_or_none,
|
||||
)
|
||||
@@ -651,37 +648,7 @@ class BaseLayerWithLoRA(nn.Module):
|
||||
self.strength = 1.0
|
||||
|
||||
|
||||
class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
|
||||
"""
|
||||
Vocab parallel embedding layer with support for LoRA (Low-Rank Adaptation).
|
||||
|
||||
Note: The current version does not yet implement the LoRA functionality.
|
||||
This class behaves exactly the same as the base VocabParallelEmbedding.
|
||||
Future versions will integrate LoRA functionality to support efficient parameter fine-tuning.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
base_layer: VocabParallelEmbedding,
|
||||
) -> None:
|
||||
super().__init__(base_layer)
|
||||
|
||||
def forward(self, input_: torch.Tensor) -> torch.Tensor:
|
||||
raise NotImplementedError(
|
||||
"We don't support VocabParallelEmbeddingWithLoRA yet."
|
||||
)
|
||||
|
||||
|
||||
class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
|
||||
def __init__(
|
||||
self,
|
||||
base_layer: ColumnParallelLinear,
|
||||
lora_rank: int | None = None,
|
||||
lora_alpha: int | None = None,
|
||||
snapshot_base: bool = True,
|
||||
) -> None:
|
||||
super().__init__(base_layer, lora_rank, lora_alpha, snapshot_base)
|
||||
|
||||
def forward(self, input_: torch.Tensor) -> torch.Tensor:
|
||||
if self.disable_lora or (self.merged and not self.has_lora_output_offset):
|
||||
return self.base_layer(input_)
|
||||
@@ -738,15 +705,6 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
|
||||
|
||||
|
||||
class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
|
||||
def __init__(
|
||||
self,
|
||||
base_layer: MergedColumnParallelLinear,
|
||||
lora_rank: int | None = None,
|
||||
lora_alpha: int | None = None,
|
||||
snapshot_base: bool = True,
|
||||
) -> None:
|
||||
super().__init__(base_layer, lora_rank, lora_alpha, snapshot_base)
|
||||
|
||||
def slice_lora_b_weights(self, B: torch.Tensor) -> torch.Tensor:
|
||||
tp_rank = get_tp_rank()
|
||||
if B.dim() == 3:
|
||||
@@ -772,15 +730,6 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
|
||||
|
||||
|
||||
class QKVParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
|
||||
def __init__(
|
||||
self,
|
||||
base_layer: QKVParallelLinear,
|
||||
lora_rank: int | None = None,
|
||||
lora_alpha: int | None = None,
|
||||
snapshot_base: bool = True,
|
||||
) -> None:
|
||||
super().__init__(base_layer, lora_rank, lora_alpha, snapshot_base)
|
||||
|
||||
def slice_lora_b_weights(
|
||||
self, B: list[torch.Tensor]
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
@@ -802,15 +751,6 @@ class QKVParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
|
||||
|
||||
|
||||
class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
|
||||
def __init__(
|
||||
self,
|
||||
base_layer: RowParallelLinear,
|
||||
lora_rank: int | None = None,
|
||||
lora_alpha: int | None = None,
|
||||
snapshot_base: bool = True,
|
||||
) -> None:
|
||||
super().__init__(base_layer, lora_rank, lora_alpha, snapshot_base)
|
||||
|
||||
def forward(self, input_: torch.Tensor):
|
||||
if self.disable_lora or (self.merged and not self.has_lora_output_offset):
|
||||
return self.base_layer(input_)
|
||||
@@ -890,15 +830,6 @@ class LinearWithLoRA(BaseLayerWithLoRA):
|
||||
not a tuple of (output, bias).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
base_layer: nn.Linear,
|
||||
lora_rank: int | None = None,
|
||||
lora_alpha: int | None = None,
|
||||
snapshot_base: bool = True,
|
||||
) -> None:
|
||||
super().__init__(base_layer, lora_rank, lora_alpha, snapshot_base)
|
||||
|
||||
@torch.compile()
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
lora_A = self.lora_A
|
||||
@@ -989,7 +920,6 @@ def wrap_with_lora_layer(
|
||||
"""
|
||||
supported_layer_types: dict[type[nn.Module], type[BaseLayerWithLoRA]] = {
|
||||
# the order matters
|
||||
# VocabParallelEmbedding: VocabParallelEmbeddingWithLoRA,
|
||||
# Weight-only FP8 LoRA is currently dynamic-only and intended for
|
||||
# single-GPU deployments.
|
||||
WeightOnlyFP8Linear: WeightOnlyFP8LinearWithLoRA,
|
||||
|
||||
@@ -181,7 +181,6 @@ def split_height_for_parallel_decode(
|
||||
def _maybe_contiguous_for_sp_gather(x: torch.Tensor) -> torch.Tensor:
|
||||
if (
|
||||
x.dim() == 5
|
||||
and hasattr(torch, "channels_last_3d")
|
||||
and x.is_contiguous(memory_format=torch.channels_last_3d)
|
||||
and not x.is_contiguous()
|
||||
):
|
||||
@@ -302,7 +301,7 @@ def gather_variable_height(
|
||||
|
||||
def _halo_memory_format(reference: torch.Tensor) -> torch.memory_format:
|
||||
if reference.dim() > 1 and reference.stride(1) == 1:
|
||||
if reference.dim() == 5 and hasattr(torch, "channels_last_3d"):
|
||||
if reference.dim() == 5:
|
||||
return torch.channels_last_3d
|
||||
if reference.dim() == 4:
|
||||
return torch.channels_last
|
||||
@@ -475,7 +474,6 @@ def _conv_preserves_local_height(
|
||||
def _conv3d_weight_is_channels_last_3d(weight: torch.Tensor) -> bool:
|
||||
return (
|
||||
weight.dim() == 5
|
||||
and hasattr(torch, "channels_last_3d")
|
||||
and (current_platform.is_cuda() or current_platform.is_rocm())
|
||||
and weight.is_contiguous(memory_format=torch.channels_last_3d)
|
||||
)
|
||||
|
||||
+14
-14
@@ -21,7 +21,10 @@ import os
|
||||
import torch
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
from sglang.multimodal_gen.runtime.layers.linear import LinearMethodBase
|
||||
from sglang.multimodal_gen.runtime.layers.linear import (
|
||||
LinearMethodBase,
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import (
|
||||
ConvRotInt8Config,
|
||||
)
|
||||
@@ -118,20 +121,17 @@ class ConvRotInt8ComfyKitchenLinearMethod(LinearMethodBase):
|
||||
"--quantization-ignored-layers"
|
||||
)
|
||||
|
||||
# The online path initially matches UnquantizedLinearMethod so the
|
||||
# source weights load in BF16 before quantization. Serialized weights
|
||||
# allocate their final INT8 storage immediately.
|
||||
weight = Parameter(
|
||||
torch.empty(
|
||||
sum(output_partition_sizes),
|
||||
input_size_per_partition,
|
||||
dtype=(torch.int8 if self.is_checkpoint_serialized else params_dtype),
|
||||
),
|
||||
requires_grad=False,
|
||||
# Online checkpoints load their original dtype; serialized ones use INT8.
|
||||
UnquantizedLinearMethod.create_weights(
|
||||
self,
|
||||
layer,
|
||||
input_size_per_partition,
|
||||
output_partition_sizes,
|
||||
input_size,
|
||||
output_size,
|
||||
torch.int8 if self.is_checkpoint_serialized else params_dtype,
|
||||
**extra_weight_attrs,
|
||||
)
|
||||
set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0})
|
||||
layer.register_parameter("weight", weight)
|
||||
set_weight_attrs(weight, extra_weight_attrs)
|
||||
if self.is_checkpoint_serialized:
|
||||
weight_scale = Parameter(
|
||||
torch.empty(
|
||||
|
||||
@@ -43,6 +43,7 @@ from sglang.multimodal_gen.runtime.layers.linear import (
|
||||
LinearMethodBase,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import (
|
||||
ConvRotInt8Config,
|
||||
@@ -202,20 +203,17 @@ class ConvRotInt8JitLinearMethod(LinearMethodBase):
|
||||
f"convrot_int8 backend jit needs the output size per "
|
||||
f"partition ({sum(output_partition_sizes)}) to be a multiple of 8"
|
||||
)
|
||||
# The online path initially matches UnquantizedLinearMethod so the
|
||||
# source weights load in BF16 before quantization. Serialized weights
|
||||
# allocate their final INT8 storage immediately.
|
||||
weight = Parameter(
|
||||
torch.empty(
|
||||
sum(output_partition_sizes),
|
||||
input_size_per_partition,
|
||||
dtype=(torch.int8 if self.is_checkpoint_serialized else params_dtype),
|
||||
),
|
||||
requires_grad=False,
|
||||
# Online checkpoints load their original dtype; serialized ones use INT8.
|
||||
UnquantizedLinearMethod.create_weights(
|
||||
self,
|
||||
layer,
|
||||
input_size_per_partition,
|
||||
output_partition_sizes,
|
||||
input_size,
|
||||
output_size,
|
||||
torch.int8 if self.is_checkpoint_serialized else params_dtype,
|
||||
**extra_weight_attrs,
|
||||
)
|
||||
set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0})
|
||||
layer.register_parameter("weight", weight)
|
||||
set_weight_attrs(weight, extra_weight_attrs)
|
||||
if self.is_checkpoint_serialized:
|
||||
# Comfy stores the per-row scale as [N, 1]; the ops read N contiguous
|
||||
# floats and accept that shape unchanged.
|
||||
|
||||
@@ -21,6 +21,9 @@ from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config impor
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import (
|
||||
ConvRotInt8Config,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.modelopt_fp8 import (
|
||||
ModelOptFp8LinearMethod as StaticModelOptFp8LinearMethod,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.parameter import (
|
||||
ModelWeightParameter,
|
||||
PerTensorScaleParameter,
|
||||
@@ -28,7 +31,6 @@ from sglang.multimodal_gen.runtime.models.parameter import (
|
||||
from sglang.multimodal_gen.runtime.platforms import current_platform
|
||||
from sglang.multimodal_gen.runtime.utils.weight_attrs import set_weight_attrs
|
||||
from sglang.srt.layers.quantization.fp8_utils import (
|
||||
apply_fp8_linear,
|
||||
cutlass_fp8_supported,
|
||||
normalize_e4m3fn_to_e4m3fnuz,
|
||||
)
|
||||
@@ -578,20 +580,7 @@ class ModelOptFp8LinearMethod(LinearMethodBase):
|
||||
copy_or_rebind_param(layer, "weight_scale", processed_weight_scale)
|
||||
copy_or_rebind_param(layer, "input_scale", layer.input_scale.max())
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
return apply_fp8_linear(
|
||||
input=x,
|
||||
weight=layer.weight,
|
||||
weight_scale=layer.weight_scale,
|
||||
input_scale=layer.input_scale,
|
||||
bias=bias,
|
||||
cutlass_fp8_supported=self.cutlass_fp8_supported,
|
||||
)
|
||||
apply = StaticModelOptFp8LinearMethod.apply
|
||||
|
||||
|
||||
class ModelOptFp4LinearMethod(LinearMethodBase):
|
||||
@@ -820,33 +809,9 @@ class ModelOptFp4LinearMethod(LinearMethodBase):
|
||||
)
|
||||
|
||||
x_fp4, x_scale_interleaved = fp4_quantize(x, layer.input_scale_inv)
|
||||
weights_padding_cols = getattr(layer, "weights_padding_cols", 0)
|
||||
x_fp4 = pad_nvfp4_activation_for_cutlass(x_fp4, weights_padding_cols)
|
||||
|
||||
w = layer.weight
|
||||
w_scale_interleaved = layer.weight_scale_interleaved
|
||||
|
||||
if x_scale_interleaved.dtype == torch.uint8:
|
||||
x_scale_interleaved = x_scale_interleaved.view(torch.float8_e4m3fn)
|
||||
if w_scale_interleaved.dtype == torch.uint8:
|
||||
w_scale_interleaved = w_scale_interleaved.view(torch.float8_e4m3fn)
|
||||
fp4_gemm, flashinfer_backend = _get_fp4_gemm_op()
|
||||
if fp4_gemm is None:
|
||||
raise RuntimeError("No FP4 GEMM kernel available. Install flashinfer.")
|
||||
out = fp4_gemm(
|
||||
x_fp4,
|
||||
w.T,
|
||||
x_scale_interleaved,
|
||||
w_scale_interleaved.T,
|
||||
layer.alpha,
|
||||
output_dtype,
|
||||
backend=flashinfer_backend,
|
||||
out = apply_nvfp4_gemm_prequantized(
|
||||
layer, x_fp4, x_scale_interleaved, output_dtype, bias
|
||||
)
|
||||
|
||||
out = slice_nvfp4_output(out, output_size)
|
||||
|
||||
if bias is not None:
|
||||
out = out + bias
|
||||
return out.view(*output_shape)
|
||||
|
||||
|
||||
|
||||
@@ -44,9 +44,6 @@ logger = init_logger(__name__)
|
||||
class NPUMXFP4Config(QuantizationConfig):
|
||||
"""Config for online MXFP4 quantization on NPU (Diffusion)."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
|
||||
@classmethod
|
||||
def get_name(cls) -> str:
|
||||
return "mxfp4_npu"
|
||||
|
||||
@@ -10,7 +10,6 @@ from .mrope import NDRotaryEmbedding, _to_tuple
|
||||
|
||||
_ROPE_DICT: dict[tuple, RotaryEmbedding] = {}
|
||||
_ND_ROPE_CACHE: "OrderedDict[tuple, NDRotaryEmbedding]" = OrderedDict()
|
||||
_ROPE_3D_CACHE: "OrderedDict[tuple, tuple[torch.Tensor, torch.Tensor]]" = OrderedDict()
|
||||
|
||||
|
||||
def get_rope(
|
||||
|
||||
@@ -367,6 +367,26 @@ class NDRotaryEmbedding(torch.nn.Module):
|
||||
positions = torch.tensor(pos_tuple, dtype=torch.long, device=device)
|
||||
return self.forward_uncached(pos=positions)
|
||||
|
||||
def forward_3d_sequence_shard(
|
||||
self,
|
||||
local_len: int,
|
||||
rank: int,
|
||||
frame_stride_local: int,
|
||||
width_local: int,
|
||||
device: torch.device,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Embed a contiguous shard of a flattened (time, height, width) grid."""
|
||||
token_start = rank * local_len
|
||||
token_indices = torch.arange(
|
||||
token_start, token_start + local_len, device=device, dtype=torch.long
|
||||
)
|
||||
t_idx = token_indices // frame_stride_local
|
||||
rem = token_indices % frame_stride_local
|
||||
h_idx = rem // width_local
|
||||
w_idx = rem % width_local
|
||||
positions = torch.stack((t_idx, h_idx, w_idx), dim=1)
|
||||
return self.forward_uncached(positions)
|
||||
|
||||
def forward_uncached(self, pos: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
The core implementation that computes embeddings from a position tensor.
|
||||
|
||||
@@ -6,7 +6,6 @@ from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
import torch.distributed._functional_collectives as ft_c
|
||||
from torch.distributed.tensor.experimental._attention import _cp_options
|
||||
|
||||
from sglang.kernels.ops.diffusion import pack_qkv_destination_major, usp_merge_heads
|
||||
@@ -28,16 +27,6 @@ if TYPE_CHECKING:
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _maybe_wait(tensor: torch.Tensor) -> torch.Tensor:
|
||||
"""
|
||||
When tracing the code, the result tensor is not an AsyncCollectiveTensor,
|
||||
so we cannot call ``wait()``.
|
||||
"""
|
||||
if isinstance(tensor, ft_c.AsyncCollectiveTensor):
|
||||
return tensor.wait()
|
||||
return tensor
|
||||
|
||||
|
||||
_A2A_STAGING_BUFFERS: dict[tuple[str, torch.dtype, int], torch.Tensor] = {}
|
||||
|
||||
|
||||
|
||||
@@ -79,13 +79,7 @@ def direct_register_custom_op(
|
||||
# Operator doesn't exist, proceed with registration
|
||||
pass
|
||||
|
||||
if hasattr(torch.library, "infer_schema"):
|
||||
schema_str = torch.library.infer_schema(op_func, mutates_args=mutates_args)
|
||||
else:
|
||||
# for pytorch 2.4
|
||||
import torch._custom_op.impl
|
||||
|
||||
schema_str = torch._custom_op.impl.infer_schema(op_func, mutates_args)
|
||||
schema_str = torch.library.infer_schema(op_func, mutates_args=mutates_args)
|
||||
|
||||
try:
|
||||
my_lib.define(op_name + schema_str)
|
||||
@@ -106,9 +100,6 @@ def direct_register_custom_op(
|
||||
else:
|
||||
# Re-raise other RuntimeErrors
|
||||
raise error
|
||||
except AttributeError as error:
|
||||
# Always re-raise AttributeError as it indicates missing dependencies
|
||||
raise error
|
||||
|
||||
|
||||
class CustomOpWrapper:
|
||||
|
||||
@@ -15,6 +15,7 @@ from sglang.multimodal_gen.runtime.distributed import (
|
||||
get_tp_group,
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.linear import UnquantizedLinearMethod
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config import (
|
||||
QuantizationConfig,
|
||||
QuantizeMethodBase,
|
||||
@@ -23,7 +24,6 @@ from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config impor
|
||||
from sglang.multimodal_gen.runtime.layers.utils import get_group_rank, get_group_size
|
||||
from sglang.multimodal_gen.runtime.models.parameter import BasevLLMParameter
|
||||
from sglang.multimodal_gen.runtime.platforms import current_platform
|
||||
from sglang.multimodal_gen.runtime.utils.weight_attrs import set_weight_attrs
|
||||
|
||||
DEFAULT_VOCAB_PADDING_SIZE = 64
|
||||
|
||||
@@ -31,29 +31,7 @@ DEFAULT_VOCAB_PADDING_SIZE = 64
|
||||
class UnquantizedEmbeddingMethod(QuantizeMethodBase):
|
||||
"""Unquantized method for embeddings."""
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
input_size_per_partition: int,
|
||||
output_partition_sizes: list[int],
|
||||
input_size: int,
|
||||
output_size: int,
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
):
|
||||
"""Create weights for embedding layer."""
|
||||
|
||||
weight = Parameter(
|
||||
torch.empty(
|
||||
sum(output_partition_sizes),
|
||||
input_size_per_partition,
|
||||
dtype=params_dtype,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0})
|
||||
layer.register_parameter("weight", weight)
|
||||
set_weight_attrs(weight, extra_weight_attrs)
|
||||
create_weights = UnquantizedLinearMethod.create_weights
|
||||
|
||||
def apply(
|
||||
self, layer: torch.nn.Module, x: torch.Tensor, bias: torch.Tensor | None = None
|
||||
@@ -344,71 +322,6 @@ class VocabParallelEmbedding(torch.nn.Module):
|
||||
added_vocab_end_index,
|
||||
)
|
||||
|
||||
def get_sharded_to_full_mapping(self) -> list[int] | None:
|
||||
"""Get a mapping that can be used to reindex the gathered
|
||||
logits for sampling.
|
||||
|
||||
During sampling, we gather logits from all ranks. The relationship
|
||||
of index->token_id will follow the same format as outlined in the class
|
||||
docstring. However, after the gather, we want to reindex the final
|
||||
logits tensor to map index->token_id one-to-one (the index is always
|
||||
equal the token_id it corresponds to). The indices returned by this
|
||||
method allow us to do that.
|
||||
"""
|
||||
if self.tp_size < 2:
|
||||
return None
|
||||
|
||||
base_embeddings: list[int] = []
|
||||
added_embeddings: list[int] = []
|
||||
padding: list[int] = []
|
||||
for tp_rank in range(self.tp_size):
|
||||
shard_indices = self._get_indices(
|
||||
self.num_embeddings_padded,
|
||||
self.org_vocab_size_padded,
|
||||
self.num_embeddings,
|
||||
self.org_vocab_size,
|
||||
tp_rank,
|
||||
self.tp_size,
|
||||
)
|
||||
range_start = self.num_embeddings_per_partition * tp_rank
|
||||
range_end = self.num_embeddings_per_partition * (tp_rank + 1)
|
||||
base_embeddings.extend(
|
||||
range(range_start, range_start + shard_indices.num_org_elements)
|
||||
)
|
||||
padding.extend(
|
||||
range(
|
||||
range_start + shard_indices.num_org_elements,
|
||||
range_start + shard_indices.num_org_elements_padded,
|
||||
)
|
||||
)
|
||||
added_embeddings.extend(
|
||||
range(
|
||||
range_start + shard_indices.num_org_elements_padded,
|
||||
range_start
|
||||
+ shard_indices.num_org_elements_padded
|
||||
+ shard_indices.num_added_elements,
|
||||
)
|
||||
)
|
||||
padding.extend(
|
||||
range(
|
||||
range_start
|
||||
+ shard_indices.num_org_elements_padded
|
||||
+ shard_indices.num_added_elements,
|
||||
range_start
|
||||
+ shard_indices.num_org_elements_padded
|
||||
+ shard_indices.num_added_elements_padded,
|
||||
)
|
||||
)
|
||||
assert (
|
||||
range_start
|
||||
+ shard_indices.num_org_elements_padded
|
||||
+ shard_indices.num_added_elements_padded
|
||||
== range_end
|
||||
)
|
||||
ret = base_embeddings + added_embeddings + padding
|
||||
assert len(ret) == self.num_embeddings_padded
|
||||
return ret
|
||||
|
||||
def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor):
|
||||
output_dim = getattr(param, "output_dim", None)
|
||||
packed_dim = getattr(param, "packed_dim", None)
|
||||
|
||||
+2
-3
@@ -5,7 +5,7 @@ from collections.abc import Generator
|
||||
import torch
|
||||
import transformers
|
||||
from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
from transformers import AutoConfig, PretrainedConfig
|
||||
from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
|
||||
|
||||
from sglang.multimodal_gen.configs.models import EncoderConfig
|
||||
@@ -83,7 +83,6 @@ from sglang.multimodal_gen.runtime.models.registry import ModelRegistry
|
||||
from sglang.multimodal_gen.runtime.platforms import current_platform
|
||||
from sglang.multimodal_gen.runtime.server_args import ServerArgs
|
||||
from sglang.multimodal_gen.runtime.utils.hf_diffusers_utils import (
|
||||
get_config,
|
||||
get_diffusers_component_config,
|
||||
load_dict,
|
||||
)
|
||||
@@ -568,7 +567,7 @@ class TextEncoderLoader(OnlineQuantizationComponentLoader):
|
||||
server_args: ServerArgs,
|
||||
component_name: str,
|
||||
) -> EncoderConfig:
|
||||
diffusers_pretrained_config = get_config(
|
||||
diffusers_pretrained_config = AutoConfig.from_pretrained(
|
||||
component_model_path, trust_remote_code=True
|
||||
)
|
||||
encoder_index = self._extract_encoder_index(
|
||||
|
||||
@@ -118,8 +118,6 @@ def _convert_conv3d_weights_to_channels_last_3d(module: nn.Module) -> int:
|
||||
Convert Conv3d weights to channels_last_3d (NDHWC) memory format.
|
||||
Returns the number of Conv3d modules converted.
|
||||
"""
|
||||
if not hasattr(torch, "channels_last_3d"):
|
||||
return 0
|
||||
num_converted = 0
|
||||
for m in module.modules():
|
||||
if isinstance(m, nn.Conv3d):
|
||||
|
||||
@@ -5,7 +5,6 @@
|
||||
"""Utilities for downloading, loading, initializing and verifying model weights."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
from collections import defaultdict
|
||||
@@ -139,38 +138,6 @@ def get_lock(model_name_or_path: str | Path, cache_dir: str | None = None):
|
||||
return lock
|
||||
|
||||
|
||||
# For models like Mistral-7B-v0.3, there are both sharded
|
||||
# safetensors files and a consolidated safetensors file.
|
||||
# Passing both of these to the weight loader functionality breaks.
|
||||
# So, we use the index_file to
|
||||
# look up which safetensors files should be used.
|
||||
def filter_duplicate_safetensors_files(
|
||||
hf_weights_files: list[str],
|
||||
hf_folder: str,
|
||||
index_file: str,
|
||||
key_filter: Callable[[str], bool] | None = None,
|
||||
) -> list[str]:
|
||||
# model.safetensors.index.json is a mapping from keys in the
|
||||
# torch state_dict to safetensors file holding that weight.
|
||||
index_file_name = os.path.join(hf_folder, index_file)
|
||||
if not os.path.isfile(index_file_name):
|
||||
return hf_weights_files
|
||||
|
||||
# Iterate through the weight_map (weight_name: safetensors files)
|
||||
# to identify weights that we should use.
|
||||
with open(index_file_name) as f:
|
||||
weight_map = json.load(f)["weight_map"]
|
||||
weight_files_in_index = set()
|
||||
for weight_name in weight_map:
|
||||
# remove only shards whose indexed tensors are all filtered
|
||||
if key_filter is not None and not key_filter(weight_name):
|
||||
continue
|
||||
weight_files_in_index.add(os.path.join(hf_folder, weight_map[weight_name]))
|
||||
# Filter out any fields that are not found in the index file.
|
||||
hf_weights_files = [f for f in hf_weights_files if f in weight_files_in_index]
|
||||
return hf_weights_files
|
||||
|
||||
|
||||
def filter_files_not_needed_for_inference(hf_weights_files: list[str]) -> list[str]:
|
||||
"""
|
||||
Exclude files that are not needed for inference.
|
||||
@@ -376,23 +343,18 @@ def pt_weights_iterator(
|
||||
|
||||
def default_weight_loader(param: torch.Tensor, loaded_weight: torch.Tensor) -> None:
|
||||
"""Default weight loader."""
|
||||
try:
|
||||
if param.numel() == 1 and loaded_weight.numel() == 1:
|
||||
# Sometimes scalar values aren't considered tensors with shapes
|
||||
# so if both param and loaded_weight are a scalar,
|
||||
# "broadcast" instead of copy
|
||||
param.data.fill_(loaded_weight.item())
|
||||
else:
|
||||
assert param.size() == loaded_weight.size(), (
|
||||
f"Attempted to load weight ({loaded_weight.size()}) "
|
||||
f"into parameter ({param.size()})"
|
||||
)
|
||||
if param.numel() == 1 and loaded_weight.numel() == 1:
|
||||
# Sometimes scalar values aren't considered tensors with shapes
|
||||
# so if both param and loaded_weight are a scalar,
|
||||
# "broadcast" instead of copy
|
||||
param.data.fill_(loaded_weight.item())
|
||||
else:
|
||||
assert param.size() == loaded_weight.size(), (
|
||||
f"Attempted to load weight ({loaded_weight.size()}) "
|
||||
f"into parameter ({param.size()})"
|
||||
)
|
||||
|
||||
param.data.copy_(loaded_weight)
|
||||
except Exception:
|
||||
# NOTE: This exception is added for the purpose of setting breakpoint to
|
||||
# debug weight loading issues.
|
||||
raise
|
||||
param.data.copy_(loaded_weight)
|
||||
|
||||
|
||||
def load_stacked_weight(
|
||||
@@ -485,6 +447,36 @@ def maybe_remap_kv_scale_name(name: str, params_dict: dict) -> str | None:
|
||||
return name
|
||||
|
||||
|
||||
def load_llm_encoder_weights(
|
||||
weights: Iterable[tuple[str, torch.Tensor]],
|
||||
params_dict: dict[str, torch.nn.Parameter],
|
||||
stacked_params_mapping: Iterable[tuple[str, str, str | int]],
|
||||
*,
|
||||
strip_prefix: str = "",
|
||||
) -> set[str]:
|
||||
"""Load Llama/Qwen-style encoder weights, excluding rotary cache tensors."""
|
||||
loaded_params: set[str] = set()
|
||||
for name, loaded_weight in weights:
|
||||
if strip_prefix and name.startswith(strip_prefix):
|
||||
name = name[len(strip_prefix) :]
|
||||
if (
|
||||
"rotary_emb.inv_freq" in name
|
||||
or "rotary_emb.cos_cached" in name
|
||||
or "rotary_emb.sin_cached" in name
|
||||
):
|
||||
continue
|
||||
if "scale" in name:
|
||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if name is None:
|
||||
continue
|
||||
name = load_stacked_weight(
|
||||
name, loaded_weight, params_dict, stacked_params_mapping
|
||||
)
|
||||
if name is not None:
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
|
||||
|
||||
def compute_weights_checksum(
|
||||
named_params: Iterable[tuple[str, torch.Tensor]],
|
||||
) -> str:
|
||||
|
||||
@@ -2,51 +2,20 @@
|
||||
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Adapted from vllm: https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/forward_context.py
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Optional, Type
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.multimodal_gen.runtime.layers.attention import AttentionMetadata
|
||||
from sglang.multimodal_gen.runtime.pipelines_core import Req
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# TODO(will): check if this is needed
|
||||
# track_batchsize: bool = envs.SGLANG_DIFFUSION_LOG_BATCHSIZE_INTERVAL >= 0
|
||||
track_batchsize: bool = False
|
||||
last_logging_time: float = 0
|
||||
forward_start_time: float = 0
|
||||
# batchsize_logging_interval: float = envs.SGLANG_DIFFUSION_LOG_BATCHSIZE_INTERVAL
|
||||
batchsize_logging_interval: float = 1000
|
||||
batchsize_forward_time: defaultdict = defaultdict(list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ForwardContext:
|
||||
current_timestep: int
|
||||
# TODO(will): check this arg
|
||||
# copy from vllm_config.compilation_config.static_forward_context
|
||||
# attn_layers: Dict[str, Any]
|
||||
# TODO: extend to support per-layer dynamic forward context
|
||||
attn_metadata: "AttentionMetadata" # set dynamically for each forward pass
|
||||
forward_batch: Optional["Req"] = None
|
||||
attention_backend_cls: Optional[Type] = None
|
||||
|
||||
def set_attn_backend_cls(self, attention_backend_cls: Type):
|
||||
if self.attention_backend_cls:
|
||||
if self.attention_backend_cls != attention_backend_cls:
|
||||
raise RuntimeError(
|
||||
f"Different types of attention backend in a same context detected, previous: {self.attention_backend_cls}, new: {attention_backend_cls}"
|
||||
)
|
||||
else:
|
||||
self.attention_backend_cls = attention_backend_cls
|
||||
|
||||
|
||||
_forward_context: Optional["ForwardContext"] = None
|
||||
@@ -65,7 +34,6 @@ def get_forward_context_or_none() -> "ForwardContext | None":
|
||||
return _forward_context
|
||||
|
||||
|
||||
# TODO(will): finalize the interface
|
||||
@contextmanager
|
||||
def set_forward_context(
|
||||
current_timestep, attn_metadata, forward_batch: Optional["Req"] = None
|
||||
@@ -74,10 +42,6 @@ def set_forward_context(
|
||||
can be attention metadata, etc.
|
||||
Here we can inject common logic for every model forward pass.
|
||||
"""
|
||||
global forward_start_time
|
||||
need_to_track_batchsize = track_batchsize and attn_metadata is not None
|
||||
if need_to_track_batchsize:
|
||||
forward_start_time = time.perf_counter()
|
||||
global _forward_context
|
||||
prev_context = _forward_context
|
||||
_forward_context = ForwardContext(
|
||||
@@ -89,36 +53,4 @@ def set_forward_context(
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
global last_logging_time, batchsize_logging_interval
|
||||
if need_to_track_batchsize:
|
||||
if hasattr(attn_metadata, "num_prefill_tokens"):
|
||||
# for v0 attention backends
|
||||
batchsize = (
|
||||
attn_metadata.num_prefill_tokens + attn_metadata.num_decode_tokens
|
||||
)
|
||||
else:
|
||||
# for v1 attention backends
|
||||
batchsize = attn_metadata.num_input_tokens
|
||||
now = time.perf_counter()
|
||||
# time measurement is in milliseconds
|
||||
batchsize_forward_time[batchsize].append((now - forward_start_time) * 1000)
|
||||
if now - last_logging_time > batchsize_logging_interval:
|
||||
last_logging_time = now
|
||||
forward_stats = []
|
||||
for bs, times in batchsize_forward_time.items():
|
||||
if len(times) <= 1:
|
||||
# can be cudagraph / profiling run
|
||||
continue
|
||||
medium = torch.quantile(torch.tensor(times), q=0.5).item()
|
||||
medium = round(medium, 2)
|
||||
forward_stats.append((bs, len(times), medium))
|
||||
forward_stats.sort(key=lambda x: x[1], reverse=True)
|
||||
if forward_stats:
|
||||
logger.info(
|
||||
(
|
||||
"Batchsize forward time stats "
|
||||
"(batchsize, count, median_time(ms)): %s"
|
||||
),
|
||||
forward_stats,
|
||||
)
|
||||
_forward_context = prev_context
|
||||
|
||||
@@ -1,25 +1,9 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Warmup-calibrated automatic component residency placement.
|
||||
"""Warmup memory, timing and layer-use estimates for the default workload.
|
||||
|
||||
Under ``--performance-mode auto`` with server warmup, each rank measures the
|
||||
peak GPU memory of bounded synthetic warmup requests and a low-step probe at
|
||||
the complete default serving shape, then selects a complete serving placement
|
||||
for every eligible component under the measured memory constraints.
|
||||
|
||||
When no full-shape measurement is available, the fallback estimate splits the
|
||||
measured peak into persistent weights and workload-scaled activations. Scaling
|
||||
the whole peak would multiply resident weights by the video frame/area cap
|
||||
ratio (~16x for Wan-class defaults) and residency adjustment would never trigger.
|
||||
|
||||
The planner targets the model default workload only (default resolution,
|
||||
default frames, batch=1). Larger shapes, batches, or multi-image inputs need
|
||||
explicit ``--component-residency``.
|
||||
|
||||
Loading and serving are deliberately separate placement states. The existing
|
||||
auto policy provides the initial state; when that state can complete loading
|
||||
and calibration, this module optimizes the long-lived serving state and
|
||||
validates the transition with a post-placement warmup. It does not force a
|
||||
single placement to serve two different lifecycle objectives.
|
||||
Peak-memory extrapolation separates persistent weights from workload-scaled
|
||||
activations. Phase and layer-use estimates account for stage-specific iteration
|
||||
counts rather than scaling every stage by the request's denoising steps.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -29,61 +13,15 @@ from typing import Iterable, Mapping
|
||||
|
||||
import msgspec
|
||||
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.component_residency import (
|
||||
COMPONENT_OFFLOAD,
|
||||
LAYERWISE_OFFLOAD,
|
||||
RESIDENT,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload_components import (
|
||||
is_dit_component_name,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
GIB_BYTES = 1024**3
|
||||
|
||||
# Activation memory rarely scales perfectly linearly with workload units;
|
||||
# pad the extrapolated activation part before checking the budget.
|
||||
ACTIVATION_EXTRAPOLATION_MARGIN = 1.2
|
||||
# A target-shape measurement plus the mandatory post-placement warmup justifies
|
||||
# a tighter reserve than an extrapolated estimate. Both retain an absolute
|
||||
# floor for allocator slack, shape variance, and CUDA graph or compile pools.
|
||||
MEASURED_VRAM_RESERVE_FRACTION = 0.05
|
||||
EXTRAPOLATED_VRAM_RESERVE_FRACTION = 0.10
|
||||
MIN_VRAM_RESERVE_BYTES = 4 * GIB_BYTES
|
||||
# The absolute floor is sized for datacenter cards, where either fraction can
|
||||
# dominate. On a 12 GiB card a flat 4 GiB would fence off a third of the device,
|
||||
# so cap the floor as a share of what is actually there.
|
||||
MAX_VRAM_RESERVE_FRACTION = 0.20
|
||||
|
||||
# A feasible placement is not automatically useful. Predictions inside this
|
||||
# interval are treated as latency-equivalent. The joint optimizer then avoids
|
||||
# changing strategy, minimizes device memory, preserves the faster estimate,
|
||||
# and finally minimizes HostPin. The raw estimate is already an upper bound:
|
||||
# transfer time is capped by the measured request.
|
||||
ESTIMATED_PINNED_H2D_BYTES_PER_SECOND = 24 * GIB_BYTES
|
||||
MIN_LATENCY_EQUIVALENCE_NS = 50_000_000
|
||||
MAX_LATENCY_EQUIVALENCE_NS = 100_000_000
|
||||
LATENCY_EQUIVALENCE_FRACTION = 0.01
|
||||
# The transfer model ranks feasible placements; the mandatory warmup is the
|
||||
# authority on whether a selected placement actually helped. Allow normal
|
||||
# measurement noise, but undo a round whose calibrated request is materially
|
||||
# slower than the original layout.
|
||||
POST_ADJUSTMENT_REGRESSION_FRACTION = 0.05
|
||||
|
||||
PLACEMENT_STATUS_SKIPPED = "skipped"
|
||||
PLACEMENT_STATUS_ADJUSTED = "adjusted"
|
||||
PLACEMENT_STATUS_VALIDATED = "validated"
|
||||
PLACEMENT_STATUS_ROLLED_BACK = "rolled_back"
|
||||
PLACEMENT_STATUS_ROLLBACK_FAILED = "rollback_failed"
|
||||
|
||||
|
||||
def describe_error(error: BaseException) -> str:
|
||||
"""Never-empty error text (str(AssertionError()) is "" and would be
|
||||
dropped by any truthiness filter)."""
|
||||
text = str(error)
|
||||
return f"{type(error).__name__}: {text}" if text else type(error).__name__
|
||||
|
||||
|
||||
class WarmupMemoryRecord(msgspec.Struct, frozen=True):
|
||||
@@ -113,96 +51,6 @@ class WarmupMemoryRecord(msgspec.Struct, frozen=True):
|
||||
return max(1, self.width) * max(1, self.height) * max(1, self.num_frames)
|
||||
|
||||
|
||||
class ResidencyTarget(msgspec.Struct, frozen=True):
|
||||
"""One complete target state for an auto-managed component."""
|
||||
|
||||
component_name: str
|
||||
residency_mode: str
|
||||
target_resident_weight_bytes: int
|
||||
# Estimated per-request host-to-device traffic this target removes.
|
||||
h2d_bytes_per_request: int
|
||||
# Layerwise candidates jointly choose stage-scoped GPU residency and host
|
||||
# pinning. None is used by ordinary component placement.
|
||||
target_layerwise_resident_layers: tuple[int, ...] | None = None
|
||||
target_layerwise_pinned_layers: tuple[tuple[int, ...], ...] | None = None
|
||||
pinned_host_delta_bytes: int = 0
|
||||
host_unpin_scratch_bytes: int = 0
|
||||
host_pin_scratch_bytes: int = 0
|
||||
host_materialize_scratch_bytes: int = 0
|
||||
# Signed device-memory delta while applying the placement before the
|
||||
# validation warmup. Layerwise -> resident materializes every managed
|
||||
# layer immediately; a demotion can release those bytes first and fund a
|
||||
# later materialization in the same transaction.
|
||||
device_transition_delta_bytes: int = 0
|
||||
permanent_residency: bool = False
|
||||
# Device-memory delta relative to the measured placement. A component
|
||||
# already loaded for its own phase has a different delta from phases where
|
||||
# it is absent; keeping both avoids adding the same weights twice.
|
||||
active_device_delta_bytes: int = 0
|
||||
# Delta when the component is already present because of async prefetch,
|
||||
# but is not the semantic owner of this phase.
|
||||
present_device_delta_bytes: int = 0
|
||||
inactive_device_delta_bytes: int = 0
|
||||
# None preserves the historical derived target for hand-built callers:
|
||||
# partial layerwise targets remain layerwise, every other option is
|
||||
# resident. Generated complete-state frontiers set this explicitly.
|
||||
target_residency_mode: str | None = None
|
||||
current_placement: bool = False
|
||||
target_device_weight_bytes: int = 0
|
||||
target_pinned_host_bytes: int = 0
|
||||
|
||||
def target_mode(self) -> str:
|
||||
if self.target_residency_mode is not None:
|
||||
return self.target_residency_mode
|
||||
if (
|
||||
self.target_layerwise_resident_layers is not None
|
||||
and not self.permanent_residency
|
||||
):
|
||||
return LAYERWISE_OFFLOAD
|
||||
return RESIDENT
|
||||
|
||||
def option_key(self) -> str:
|
||||
target_mode = self.target_mode()
|
||||
if target_mode == COMPONENT_OFFLOAD:
|
||||
return f"{self.component_name}:component-offload"
|
||||
if self.target_layerwise_resident_layers is None:
|
||||
return f"{self.component_name}:resident"
|
||||
layer_counts = ",".join(
|
||||
str(count) for count in self.target_layerwise_resident_layers
|
||||
)
|
||||
pinned = "|".join(
|
||||
",".join(str(index) for index in indices) or "-"
|
||||
for indices in self.target_layerwise_pinned_layers or ()
|
||||
)
|
||||
permanence = "permanent" if self.permanent_residency else "stage"
|
||||
return f"{self.component_name}:{permanence}:layers={layer_counts}:pins={pinned}"
|
||||
|
||||
|
||||
class RankResidencyReport(msgspec.Struct, frozen=True):
|
||||
"""One rank's inputs to the replica-wide placement decision."""
|
||||
|
||||
rank: int
|
||||
budget_bytes: int
|
||||
estimated_peak_bytes: int | None
|
||||
target_workload_measured: bool = False
|
||||
observed_reserved_bytes: int = 0
|
||||
estimated_peak_bytes_by_phase: dict[str, int] = {}
|
||||
active_components_by_phase: dict[str, tuple[str, ...]] = {}
|
||||
used_components_by_phase: dict[str, tuple[str, ...]] = {}
|
||||
full_weight_transition_components_by_phase: dict[str, tuple[str, ...]] = {}
|
||||
current_device_weight_bytes_by_component: dict[str, int] = {}
|
||||
node_rank: int = 0
|
||||
pinned_host_bytes: int = 0
|
||||
host_pin_capacity_bytes: int = 0
|
||||
host_transition_headroom_bytes: int = 0
|
||||
device_transition_allocated_bytes: int = 0
|
||||
estimated_request_duration_ns: int = 0
|
||||
measured_request_duration_ns: int = 0
|
||||
candidate_latency_savings_ns: dict[str, int] = {}
|
||||
candidates: list[ResidencyTarget] = []
|
||||
skip_reason: str | None = None
|
||||
|
||||
|
||||
def estimate_layerwise_layer_uses(
|
||||
*,
|
||||
records: Iterable[WarmupMemoryRecord],
|
||||
|
||||
+2
-8
@@ -12,6 +12,7 @@ import os
|
||||
from dataclasses import dataclass
|
||||
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload_components import (
|
||||
component_base_name,
|
||||
is_dit_component_name,
|
||||
is_image_encoder_component_name,
|
||||
is_text_encoder_component_name,
|
||||
@@ -34,13 +35,6 @@ class ComponentLoadSpec:
|
||||
_WEIGHT_FILE_SUFFIXES = (".bin", ".pt", ".pth")
|
||||
|
||||
|
||||
def _component_base_name(component_name: str) -> str:
|
||||
prefix, separator, suffix = component_name.rpartition("_")
|
||||
if separator and suffix.isdigit():
|
||||
return prefix
|
||||
return component_name
|
||||
|
||||
|
||||
def _component_variant_priority(component_name: str) -> int:
|
||||
_, separator, suffix = component_name.rpartition("_")
|
||||
if separator and suffix.isdigit():
|
||||
@@ -50,7 +44,7 @@ def _component_variant_priority(component_name: str) -> int:
|
||||
|
||||
def component_load_risk_rank(component_name: str) -> int:
|
||||
"""Fallback type rank when checkpoint size cannot be inferred."""
|
||||
candidate_names = (component_name, _component_base_name(component_name))
|
||||
candidate_names = (component_name, component_base_name(component_name))
|
||||
if any(is_dit_component_name(name) for name in candidate_names):
|
||||
return 0
|
||||
if any(is_text_encoder_component_name(name) for name in candidate_names):
|
||||
|
||||
@@ -98,7 +98,6 @@ class CausalWanSelfAttention(nn.Module):
|
||||
self.sink_size = sink_size
|
||||
self.qk_norm = qk_norm
|
||||
self.eps = eps
|
||||
self.parallel_attention = parallel_attention
|
||||
|
||||
# Scaled dot product attention
|
||||
self.attn = LocalAttention(
|
||||
|
||||
@@ -375,7 +375,6 @@ class DomainAwareLinear(nn.Module):
|
||||
super().__init__()
|
||||
self.input_size = input_size
|
||||
self.output_size = output_size
|
||||
self.num_domains = num_domains
|
||||
self.fc = nn.Embedding(num_domains, output_size * input_size)
|
||||
self.bias = nn.Embedding(num_domains, output_size)
|
||||
nn.init.xavier_uniform_(
|
||||
@@ -583,6 +582,27 @@ def _build_mlp(
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _initialize_attention_projections(module, prefix, quant_config):
|
||||
module.q_size = module.num_attention_heads * module.head_dim
|
||||
module.kv_size = module.num_key_value_heads * module.head_dim
|
||||
module.to_qkv = MergedColumnParallelLinear(
|
||||
module.hidden_size,
|
||||
[module.q_size, module.kv_size, module.kv_size],
|
||||
bias=False,
|
||||
gather_output=False,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("to_qkv", prefix),
|
||||
)
|
||||
module.to_out = RowParallelLinear(
|
||||
module.q_size,
|
||||
module.hidden_size,
|
||||
bias=False,
|
||||
input_is_parallel=True,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("to_out", prefix),
|
||||
)
|
||||
|
||||
|
||||
class Cosmos3CausalAttention(nn.Module):
|
||||
"""Understanding pathway: causal self-attention on text tokens."""
|
||||
|
||||
@@ -618,24 +638,7 @@ class Cosmos3CausalAttention(nn.Module):
|
||||
self.local_num_attention_heads = num_attention_heads // self.tp_size
|
||||
self.local_num_key_value_heads = num_key_value_heads // self.tp_size
|
||||
|
||||
self.q_size = num_attention_heads * head_dim
|
||||
self.kv_size = num_key_value_heads * head_dim
|
||||
self.to_qkv = MergedColumnParallelLinear(
|
||||
hidden_size,
|
||||
[self.q_size, self.kv_size, self.kv_size],
|
||||
bias=False,
|
||||
gather_output=False,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("to_qkv", prefix),
|
||||
)
|
||||
self.to_out = RowParallelLinear(
|
||||
num_attention_heads * head_dim,
|
||||
hidden_size,
|
||||
bias=False,
|
||||
input_is_parallel=True,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("to_out", prefix),
|
||||
)
|
||||
_initialize_attention_projections(self, prefix, quant_config)
|
||||
|
||||
# Per-head QK norm (optional; some backbones omit it on text).
|
||||
if qk_norm:
|
||||
@@ -754,24 +757,7 @@ class Cosmos3CrossAttention(nn.Module):
|
||||
self.local_num_attention_heads = num_attention_heads // self.tp_size
|
||||
self.local_num_key_value_heads = num_key_value_heads // self.tp_size
|
||||
|
||||
self.q_size = num_attention_heads * head_dim
|
||||
self.kv_size = num_key_value_heads * head_dim
|
||||
self.to_qkv = MergedColumnParallelLinear(
|
||||
hidden_size,
|
||||
[self.q_size, self.kv_size, self.kv_size],
|
||||
bias=False,
|
||||
gather_output=False,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("to_qkv", prefix),
|
||||
)
|
||||
self.to_out = RowParallelLinear(
|
||||
num_attention_heads * head_dim,
|
||||
hidden_size,
|
||||
bias=False,
|
||||
input_is_parallel=True,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("to_out", prefix),
|
||||
)
|
||||
_initialize_attention_projections(self, prefix, quant_config)
|
||||
|
||||
self.norm_q = RMSNorm(head_dim, eps=1e-6)
|
||||
self.norm_k = RMSNorm(head_dim, eps=1e-6)
|
||||
|
||||
@@ -523,7 +523,6 @@ class FluxAttention(torch.nn.Module, AttentionModuleMixin):
|
||||
)
|
||||
self.local_heads = divide(self.heads, self.tp_size)
|
||||
self.added_kv_proj_dim = added_kv_proj_dim
|
||||
self.added_proj_bias = added_proj_bias
|
||||
|
||||
self.use_fused_qkv = isinstance(quant_config, NunchakuConfig)
|
||||
self.use_fused_added_qkv = isinstance(quant_config, NunchakuConfig)
|
||||
|
||||
@@ -573,7 +573,6 @@ class Flux2Attention(torch.nn.Module, AttentionModuleMixin):
|
||||
self.dropout = dropout
|
||||
|
||||
self.added_kv_proj_dim = added_kv_proj_dim
|
||||
self.added_proj_bias = added_proj_bias
|
||||
|
||||
# Packed NVFP4 checkpoints already serialize QKV together. ModelOpt
|
||||
# FP8 exports separate Diffusers tensors, but the loader can merge
|
||||
|
||||
@@ -669,7 +669,6 @@ class HunyuanVideoTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixi
|
||||
self.out_channels = (
|
||||
config.in_channels if config.out_channels is None else config.out_channels
|
||||
)
|
||||
self.unpatchify_channels = self.out_channels
|
||||
self.guidance_embeds = config.guidance_embeds
|
||||
self.rope_dim_list = list(config.rope_axes_dim)
|
||||
self.rope_theta = config.rope_theta
|
||||
|
||||
@@ -704,16 +704,9 @@ class LingBotWorldTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixi
|
||||
width_local: int,
|
||||
device: torch.device,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
token_start = rank * local_len
|
||||
token_indices = torch.arange(
|
||||
token_start, token_start + local_len, device=device, dtype=torch.long
|
||||
return self.rotary_emb.forward_3d_sequence_shard(
|
||||
local_len, rank, frame_stride_local, width_local, device
|
||||
)
|
||||
t_idx = token_indices // frame_stride_local
|
||||
rem = token_indices % frame_stride_local
|
||||
h_idx = rem // width_local
|
||||
w_idx = rem % width_local
|
||||
positions = torch.stack((t_idx, h_idx, w_idx), dim=1)
|
||||
return self.rotary_emb.forward_uncached(positions)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
|
||||
@@ -520,11 +520,6 @@ class LTX2AudioVideoRotaryPosEmbed(nn.Module):
|
||||
audio_coords = audio_coords.unsqueeze(1)
|
||||
return audio_coords
|
||||
|
||||
def prepare_coords(self, *args, **kwargs):
|
||||
if self.modality == "video":
|
||||
return self.prepare_video_coords(*args, **kwargs)
|
||||
return self.prepare_audio_coords(*args, **kwargs)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
coords: torch.Tensor,
|
||||
@@ -697,7 +692,6 @@ class LTX2TPRMSNormAcrossHeads(nn.Module):
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.full_hidden_size = full_hidden_size
|
||||
self.local_hidden_size = local_hidden_size
|
||||
self.eps = eps
|
||||
self.weight = nn.Parameter(torch.ones(local_hidden_size))
|
||||
|
||||
@@ -1864,8 +1858,6 @@ class LTX2VideoTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixin):
|
||||
num_attention_heads=self.audio_num_attention_heads,
|
||||
)
|
||||
|
||||
self.cross_pe_max_pos = cross_attn_pos_embed_max_pos
|
||||
|
||||
# 5. Transformer Blocks
|
||||
self.transformer_blocks = nn.ModuleList(
|
||||
[
|
||||
|
||||
@@ -15,7 +15,6 @@ from torch.distributed.tensor import DTensor
|
||||
from sglang.multimodal_gen.configs.models.dits.mova_audio import MOVAAudioConfig
|
||||
from sglang.multimodal_gen.configs.models.fsdp import is_block
|
||||
from sglang.multimodal_gen.runtime.layers.linear import ReplicatedLinear
|
||||
from sglang.multimodal_gen.runtime.layers.mlp import MLP
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config import (
|
||||
QuantizationConfig,
|
||||
)
|
||||
@@ -25,7 +24,11 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload im
|
||||
from sglang.multimodal_gen.runtime.models.dits.base import CachableDiT
|
||||
|
||||
# Reuse common functions and classes from mova_video_dit
|
||||
from .mova_video_dit import DiTBlock, precompute_freqs_cis, sinusoidal_embedding_1d
|
||||
from .mova_video_dit import (
|
||||
initialize_mova_transformer_layers,
|
||||
precompute_freqs_cis,
|
||||
sinusoidal_embedding_1d,
|
||||
)
|
||||
|
||||
|
||||
def precompute_freqs_cis_1d(dim: int, end: int = 16384, theta: float = 10000.0):
|
||||
@@ -102,15 +105,11 @@ class WanAudioModel(CachableDiT, LayerwiseOffloadableModuleMixin):
|
||||
# Extract parameters from config
|
||||
dim = config.dim
|
||||
in_dim = config.in_dim
|
||||
ffn_dim = config.ffn_dim
|
||||
out_dim = config.out_dim
|
||||
text_dim = config.text_dim
|
||||
freq_dim = config.freq_dim
|
||||
eps = config.eps
|
||||
patch_size = config.patch_size
|
||||
num_heads = config.num_heads
|
||||
num_layers = config.num_layers
|
||||
has_image_pos_emb = config.has_image_pos_emb
|
||||
has_ref_conv = config.has_ref_conv
|
||||
separated_timestep = config.separated_timestep
|
||||
require_vae_embedding = config.require_vae_embedding
|
||||
@@ -126,38 +125,15 @@ class WanAudioModel(CachableDiT, LayerwiseOffloadableModuleMixin):
|
||||
self.require_clip_embedding = require_clip_embedding
|
||||
self.fuse_vae_embedding_in_latents = fuse_vae_embedding_in_latents
|
||||
self.vae_type = vae_type
|
||||
# self.patch_embedding = nn.Conv3d(
|
||||
# in_dim, dim, kernel_size=patch_size, stride=patch_size)
|
||||
self.patch_embedding = Conv1dLocalIsland(
|
||||
in_dim, dim, kernel_size=patch_size, stride=patch_size
|
||||
)
|
||||
self.text_embedding = MLP(
|
||||
text_dim,
|
||||
dim,
|
||||
output_dim=dim,
|
||||
act_type="gelu_pytorch_tanh",
|
||||
quant_config=quant_config,
|
||||
)
|
||||
self.time_embedding = MLP(
|
||||
freq_dim, dim, output_dim=dim, act_type="silu", quant_config=quant_config
|
||||
)
|
||||
# Preserve state_dict keys (time_projection.1.weight/bias).
|
||||
self.time_projection = nn.Sequential(
|
||||
nn.SiLU(), ReplicatedLinear(dim, dim * 6, quant_config=quant_config)
|
||||
)
|
||||
self.blocks = nn.ModuleList(
|
||||
[
|
||||
DiTBlock(dim, num_heads, ffn_dim, eps, quant_config=quant_config)
|
||||
for _ in range(num_layers)
|
||||
]
|
||||
)
|
||||
initialize_mova_transformer_layers(self, config, quant_config)
|
||||
self.head = Head(dim, out_dim, patch_size, eps)
|
||||
self.num_heads = num_heads
|
||||
self.freqs = None
|
||||
self.img_pos_emb = None
|
||||
if has_ref_conv:
|
||||
self.ref_conv = nn.Conv2d(16, dim, kernel_size=(2, 2), stride=(2, 2))
|
||||
self.has_image_pos_emb = has_image_pos_emb
|
||||
self.has_ref_conv = has_ref_conv
|
||||
self.hidden_size = dim
|
||||
self.num_attention_heads = num_heads
|
||||
|
||||
@@ -272,9 +272,6 @@ class CrossAttention(_MOVAAttention):
|
||||
|
||||
|
||||
class MulAdd(nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
||||
def forward(self, x, gate, residual):
|
||||
return residual + gate * x
|
||||
|
||||
@@ -408,6 +405,41 @@ class Conv3dLocalIsland(nn.Conv3d):
|
||||
return super().forward(input)
|
||||
|
||||
|
||||
def initialize_mova_transformer_layers(model, config, quant_config):
|
||||
"""Create shared audio/video layers in checkpoint registration order."""
|
||||
model.text_embedding = MLP(
|
||||
config.text_dim,
|
||||
config.dim,
|
||||
output_dim=config.dim,
|
||||
act_type="gelu_pytorch_tanh",
|
||||
quant_config=quant_config,
|
||||
)
|
||||
model.time_embedding = MLP(
|
||||
config.freq_dim,
|
||||
config.dim,
|
||||
output_dim=config.dim,
|
||||
act_type="silu",
|
||||
quant_config=quant_config,
|
||||
)
|
||||
# Preserve state_dict keys (time_projection.1.weight/bias).
|
||||
model.time_projection = nn.Sequential(
|
||||
nn.SiLU(),
|
||||
ReplicatedLinear(config.dim, config.dim * 6, quant_config=quant_config),
|
||||
)
|
||||
model.blocks = nn.ModuleList(
|
||||
[
|
||||
DiTBlock(
|
||||
config.dim,
|
||||
config.num_heads,
|
||||
config.ffn_dim,
|
||||
config.eps,
|
||||
quant_config=quant_config,
|
||||
)
|
||||
for _ in range(config.num_layers)
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
class WanModel(CachableDiT, LayerwiseOffloadableModuleMixin):
|
||||
_fsdp_shard_conditions = [is_block]
|
||||
_compile_conditions = [is_block]
|
||||
@@ -426,15 +458,11 @@ class WanModel(CachableDiT, LayerwiseOffloadableModuleMixin):
|
||||
# Extract parameters from config
|
||||
dim = config.dim
|
||||
in_dim = config.in_dim
|
||||
ffn_dim = config.ffn_dim
|
||||
out_dim = config.out_dim
|
||||
text_dim = config.text_dim
|
||||
freq_dim = config.freq_dim
|
||||
eps = config.eps
|
||||
patch_size = config.patch_size
|
||||
num_heads = config.num_heads
|
||||
num_layers = config.num_layers
|
||||
has_image_pos_emb = config.has_image_pos_emb
|
||||
has_ref_conv = config.has_ref_conv
|
||||
separated_timestep = config.separated_timestep
|
||||
require_vae_embedding = config.require_vae_embedding
|
||||
@@ -452,33 +480,13 @@ class WanModel(CachableDiT, LayerwiseOffloadableModuleMixin):
|
||||
self.patch_embedding = Conv3dLocalIsland(
|
||||
in_dim, dim, kernel_size=patch_size, stride=patch_size
|
||||
)
|
||||
self.text_embedding = MLP(
|
||||
text_dim,
|
||||
dim,
|
||||
output_dim=dim,
|
||||
act_type="gelu_pytorch_tanh",
|
||||
quant_config=quant_config,
|
||||
)
|
||||
self.time_embedding = MLP(
|
||||
freq_dim, dim, output_dim=dim, act_type="silu", quant_config=quant_config
|
||||
)
|
||||
# Preserve state_dict keys (time_projection.1.weight/bias).
|
||||
self.time_projection = nn.Sequential(
|
||||
nn.SiLU(), ReplicatedLinear(dim, dim * 6, quant_config=quant_config)
|
||||
)
|
||||
self.blocks = nn.ModuleList(
|
||||
[
|
||||
DiTBlock(dim, num_heads, ffn_dim, eps, quant_config=quant_config)
|
||||
for _ in range(num_layers)
|
||||
]
|
||||
)
|
||||
initialize_mova_transformer_layers(self, config, quant_config)
|
||||
self.head = Head(dim, out_dim, patch_size, eps)
|
||||
self.num_heads = num_heads
|
||||
self.freqs = None
|
||||
|
||||
if has_ref_conv:
|
||||
self.ref_conv = nn.Conv2d(16, dim, kernel_size=(2, 2), stride=(2, 2))
|
||||
self.has_image_pos_emb = has_image_pos_emb
|
||||
self.has_ref_conv = has_ref_conv
|
||||
self.hidden_size = dim
|
||||
self.num_attention_heads = num_heads
|
||||
|
||||
@@ -744,7 +744,6 @@ class QwenImageCrossAttention(nn.Module):
|
||||
self.window_size = window_size
|
||||
self.qk_norm = qk_norm
|
||||
self.eps = eps
|
||||
self.parallel_attention = parallel_attention
|
||||
self.added_kv_proj_dim = added_kv_proj_dim
|
||||
self.prefix = prefix
|
||||
self.defer_output_bias = _defer_modelopt_output_bias(quant_config)
|
||||
|
||||
@@ -169,6 +169,17 @@ class SanaWMBlock(nn.Module):
|
||||
tokens_per_frame = N // num_frames
|
||||
return x.reshape(B, num_frames, tokens_per_frame, C), tokens_per_frame
|
||||
|
||||
def _prepare_modulation(self, t: torch.Tensor, batch_size: int):
|
||||
if t.dim() == 2:
|
||||
return None, (
|
||||
self.scale_shift_table[None] + t.reshape(batch_size, 6, -1)
|
||||
).chunk(6, dim=1)
|
||||
num_frames = t.reshape(batch_size, -1, 6, t.shape[-1] // 6).shape[1]
|
||||
t = t.reshape(batch_size, num_frames, 6, -1)
|
||||
return num_frames, (self.scale_shift_table[None, None, :, :] + t).chunk(
|
||||
6, dim=2
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
x: torch.Tensor, # (B, N, D)
|
||||
@@ -184,17 +195,8 @@ class SanaWMBlock(nn.Module):
|
||||
chunk_index: Optional[List[int]] = None,
|
||||
) -> torch.Tensor:
|
||||
B = x.shape[0]
|
||||
if t.dim() == 2:
|
||||
num_frames = None
|
||||
shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = (
|
||||
self.scale_shift_table[None] + t.reshape(B, 6, -1)
|
||||
).chunk(6, dim=1)
|
||||
else:
|
||||
num_frames = t.reshape(B, -1, 6, t.shape[-1] // 6).shape[1]
|
||||
t = t.reshape(B, num_frames, 6, -1)
|
||||
shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = (
|
||||
self.scale_shift_table[None, None, :, :] + t
|
||||
).chunk(6, dim=2)
|
||||
num_frames, modulation = self._prepare_modulation(t, B)
|
||||
shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = modulation
|
||||
|
||||
# Self-attention with UCPE camera branch
|
||||
if num_frames is None:
|
||||
@@ -259,17 +261,8 @@ class SanaWMBlock(nn.Module):
|
||||
) -> Tuple[torch.Tensor, list]:
|
||||
"""Streaming counterpart of ``forward``: threads the per-block 10-slot ``kv_cache`` through cached attention + FFN."""
|
||||
B = x.shape[0]
|
||||
if t.dim() == 2:
|
||||
num_frames = None
|
||||
shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = (
|
||||
self.scale_shift_table[None] + t.reshape(B, 6, -1)
|
||||
).chunk(6, dim=1)
|
||||
else:
|
||||
num_frames = t.reshape(B, -1, 6, t.shape[-1] // 6).shape[1]
|
||||
t = t.reshape(B, num_frames, 6, -1)
|
||||
shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = (
|
||||
self.scale_shift_table[None, None, :, :] + t
|
||||
).chunk(6, dim=2)
|
||||
num_frames, modulation = self._prepare_modulation(t, B)
|
||||
shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = modulation
|
||||
|
||||
if num_frames is None:
|
||||
x_in = self._modulate(self.norm1(x), shift_msa, scale_msa)
|
||||
@@ -435,7 +428,6 @@ class SanaWMTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixin):
|
||||
for i in range(depth)
|
||||
if arch.softmax_every_n > 0 and (i + 1) % arch.softmax_every_n == 0
|
||||
)
|
||||
self.softmax_block_indices = tuple(sorted(softmax_idx))
|
||||
|
||||
self.blocks = nn.ModuleList(
|
||||
[
|
||||
|
||||
@@ -203,7 +203,6 @@ class WanSelfAttention(nn.Module):
|
||||
self.window_size = window_size
|
||||
self.qk_norm = qk_norm
|
||||
self.eps = eps
|
||||
self.parallel_attention = parallel_attention
|
||||
tp_size = get_tp_world_size()
|
||||
|
||||
# layers
|
||||
@@ -1137,19 +1136,9 @@ class WanTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixin):
|
||||
width_local: int,
|
||||
device: torch.device,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
token_start = rank * local_len
|
||||
token_indices = torch.arange(
|
||||
token_start,
|
||||
token_start + local_len,
|
||||
device=device,
|
||||
dtype=torch.long,
|
||||
return self.rotary_emb.forward_3d_sequence_shard(
|
||||
local_len, rank, frame_stride_local, width_local, device
|
||||
)
|
||||
t_idx = token_indices // frame_stride_local
|
||||
rem = token_indices % frame_stride_local
|
||||
h_idx = rem // width_local
|
||||
w_idx = rem % width_local
|
||||
positions = torch.stack((t_idx, h_idx, w_idx), dim=1)
|
||||
return self.rotary_emb.forward_uncached(positions)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
|
||||
@@ -1379,10 +1379,6 @@ class ZImageTransformer2DModel(CachableDiT, LayerwiseOffloadableModuleMixin):
|
||||
self._cached_joint_attn_mask_meta = (cache_key, result)
|
||||
return self._pin_for_active_capture(result)
|
||||
|
||||
@staticmethod
|
||||
def _has_padding(valid_lens: list[int], target_len: int) -> bool:
|
||||
return any(int(length) < target_len for length in valid_lens)
|
||||
|
||||
@staticmethod
|
||||
def _as_image_list(hidden_states) -> list[torch.Tensor]:
|
||||
"""Normalize 4D/5D image latents into per-sample tensors."""
|
||||
|
||||
@@ -31,6 +31,18 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload im
|
||||
from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
|
||||
|
||||
|
||||
def get_attention_head_partition(
|
||||
num_heads: int, num_kv_heads: int, tp_size: int
|
||||
) -> tuple[int, int]:
|
||||
"""Partition query heads and shard or replicate KV heads across TP ranks."""
|
||||
assert num_heads % tp_size == 0
|
||||
if num_kv_heads >= tp_size:
|
||||
assert num_kv_heads % tp_size == 0
|
||||
else:
|
||||
assert tp_size % num_kv_heads == 0
|
||||
return num_heads // tp_size, max(1, num_kv_heads // tp_size)
|
||||
|
||||
|
||||
def get_folding_tp_group(config: EncoderConfig):
|
||||
"""Return the TP group selected for an encoder."""
|
||||
mode = config.parallel_folding_mode
|
||||
@@ -218,7 +230,6 @@ class TextEncoder(
|
||||
# shard conditions left in the root group stays sharded unless the entry
|
||||
# point is registered; loaders read this and register each name.
|
||||
_fsdp_forward_methods: tuple[str, ...] = ()
|
||||
_stacked_params_mapping: list[tuple[str, str, str]] = field(default_factory=list)
|
||||
_supported_attention_backends: set[AttentionBackendEnum] = (
|
||||
TextEncoderConfig()._supported_attention_backends
|
||||
)
|
||||
@@ -227,7 +238,6 @@ class TextEncoder(
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self._fsdp_shard_conditions = config.arch_config._fsdp_shard_conditions
|
||||
self._stacked_params_mapping = config.arch_config.stacked_params_mapping
|
||||
if not self.supported_attention_backends:
|
||||
raise ValueError(
|
||||
f"Subclass {self.__class__.__name__} must define _supported_attention_backends"
|
||||
|
||||
@@ -40,6 +40,7 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload im
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.encoders.base import (
|
||||
EncoderTensorParallelMixin,
|
||||
get_attention_head_partition,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -114,14 +115,10 @@ class Gemma2Attention(nn.Module):
|
||||
self.hidden_size = hidden_size
|
||||
tp_size = get_tp_world_size()
|
||||
self.total_num_heads = num_heads
|
||||
assert self.total_num_heads % tp_size == 0
|
||||
self.num_heads = self.total_num_heads // tp_size
|
||||
self.total_num_kv_heads = num_kv_heads
|
||||
if self.total_num_kv_heads >= tp_size:
|
||||
assert self.total_num_kv_heads % tp_size == 0
|
||||
else:
|
||||
assert tp_size % self.total_num_kv_heads == 0
|
||||
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
|
||||
self.num_heads, self.num_kv_heads = get_attention_head_partition(
|
||||
num_heads, num_kv_heads, tp_size
|
||||
)
|
||||
|
||||
arch = config.arch_config
|
||||
self.head_dim = arch.head_dim
|
||||
|
||||
@@ -27,6 +27,7 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload im
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.encoders.base import (
|
||||
EncoderTensorParallelMixin,
|
||||
get_attention_head_partition,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.utils.common import add_prefix
|
||||
from sglang.srt.models.siglip import SiglipVisionModel
|
||||
@@ -68,6 +69,36 @@ def get_attention_sliding_window_size(config):
|
||||
return config.sliding_window - 1
|
||||
|
||||
|
||||
def _load_with_shard_id(
|
||||
weight_loader, param, loaded_weight: torch.Tensor, shard_id
|
||||
) -> None:
|
||||
"""Try the supplied shard ID, then its equivalent q/k/v or integer form."""
|
||||
try:
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
return
|
||||
except (AssertionError, TypeError):
|
||||
pass
|
||||
|
||||
if isinstance(shard_id, str):
|
||||
mapping = {"q": 0, "k": 1, "v": 2}
|
||||
if shard_id in mapping:
|
||||
weight_loader(param, loaded_weight, mapping[shard_id])
|
||||
return
|
||||
if shard_id.isdigit():
|
||||
weight_loader(param, loaded_weight, int(shard_id))
|
||||
return
|
||||
elif isinstance(shard_id, int):
|
||||
mapping = {0: "q", 1: "k", 2: "v"}
|
||||
if shard_id in mapping:
|
||||
weight_loader(param, loaded_weight, mapping[shard_id])
|
||||
return
|
||||
|
||||
raise TypeError(
|
||||
f"Unsupported shard_id={shard_id!r} for weight_loader={weight_loader} "
|
||||
f"(param={getattr(param, 'name', '<param>')})."
|
||||
)
|
||||
|
||||
|
||||
class Gemma3RMSNorm(nn.Module):
|
||||
def __init__(self, dim: int, eps: float = 1e-6):
|
||||
super().__init__()
|
||||
@@ -146,14 +177,10 @@ class Gemma3Attention(nn.Module):
|
||||
self.hidden_size = hidden_size
|
||||
tp_size = get_tp_world_size()
|
||||
self.total_num_heads = num_heads
|
||||
assert self.total_num_heads % tp_size == 0
|
||||
self.num_heads = self.total_num_heads // tp_size
|
||||
self.total_num_kv_heads = num_kv_heads
|
||||
if self.total_num_kv_heads >= tp_size:
|
||||
assert self.total_num_kv_heads % tp_size == 0
|
||||
else:
|
||||
assert tp_size % self.total_num_kv_heads == 0
|
||||
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
|
||||
self.num_heads, self.num_kv_heads = get_attention_head_partition(
|
||||
num_heads, num_kv_heads, tp_size
|
||||
)
|
||||
|
||||
self.head_dim = getattr(
|
||||
config.text_config, "head_dim", self.hidden_size // self.total_num_heads
|
||||
|
||||
@@ -51,10 +51,12 @@ from sglang.multimodal_gen.runtime.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.loader.weight_utils import (
|
||||
load_stacked_weight,
|
||||
maybe_remap_kv_scale_name,
|
||||
load_llm_encoder_weights,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.encoders.base import (
|
||||
TextEncoder,
|
||||
get_attention_head_partition,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.encoders.base import TextEncoder
|
||||
|
||||
|
||||
class LlamaMLP(nn.Module):
|
||||
@@ -116,18 +118,10 @@ class LlamaAttention(nn.Module):
|
||||
self.hidden_size = hidden_size
|
||||
tp_size = get_tp_world_size()
|
||||
self.total_num_heads = num_heads
|
||||
assert self.total_num_heads % tp_size == 0
|
||||
self.num_heads = self.total_num_heads // tp_size
|
||||
self.total_num_kv_heads = num_kv_heads
|
||||
if self.total_num_kv_heads >= tp_size:
|
||||
# Number of KV heads is greater than TP size, so we partition
|
||||
# the KV heads across multiple tensor parallel GPUs.
|
||||
assert self.total_num_kv_heads % tp_size == 0
|
||||
else:
|
||||
# Number of KV heads is less than TP size, so we replicate
|
||||
# the KV heads across multiple tensor parallel GPUs.
|
||||
assert tp_size % self.total_num_kv_heads == 0
|
||||
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
|
||||
self.num_heads, self.num_kv_heads = get_attention_head_partition(
|
||||
num_heads, num_kv_heads, tp_size
|
||||
)
|
||||
# MistralConfig has an optional head_dim introduced by Mistral-Nemo
|
||||
self.head_dim = getattr(
|
||||
config, "head_dim", self.hidden_size // self.total_num_heads
|
||||
@@ -390,32 +384,11 @@ class LlamaModel(TextEncoder):
|
||||
return output
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
for name, loaded_weight in weights:
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
continue
|
||||
if "rotary_emb.cos_cached" in name or "rotary_emb.sin_cached" in name:
|
||||
# Models trained using ColossalAI may include these tensors in
|
||||
# the checkpoint. Skip them.
|
||||
continue
|
||||
if "scale" in name:
|
||||
# Remapping the name of FP8 kv-scale.
|
||||
kv_scale_name: str | None = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if kv_scale_name is None:
|
||||
continue
|
||||
else:
|
||||
name = kv_scale_name
|
||||
name = load_stacked_weight(
|
||||
name,
|
||||
loaded_weight,
|
||||
params_dict,
|
||||
self.config.arch_config.stacked_params_mapping,
|
||||
)
|
||||
if name is not None:
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
return load_llm_encoder_weights(
|
||||
weights,
|
||||
dict(self.named_parameters()),
|
||||
self.config.arch_config.stacked_params_mapping,
|
||||
)
|
||||
|
||||
|
||||
EntryClass = LlamaModel
|
||||
|
||||
@@ -1332,13 +1332,10 @@ class Qwen2_5_VLForConditionalGeneration(TextEncoder):
|
||||
break
|
||||
if loaded_stacked_param:
|
||||
continue
|
||||
try:
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
param = params_dict[name]
|
||||
except KeyError:
|
||||
raise
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
param = params_dict[name]
|
||||
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
loaded_weight = loaded_weight.to(param.dtype)
|
||||
|
||||
@@ -20,10 +20,12 @@ from sglang.multimodal_gen.runtime.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.loader.weight_utils import (
|
||||
load_stacked_weight,
|
||||
maybe_remap_kv_scale_name,
|
||||
load_llm_encoder_weights,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.encoders.base import (
|
||||
TextEncoder,
|
||||
get_attention_head_partition,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.encoders.base import TextEncoder
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
|
||||
@@ -91,14 +93,10 @@ class Qwen3Attention(nn.Module):
|
||||
self.hidden_size = hidden_size
|
||||
tp_size = get_tp_world_size()
|
||||
self.total_num_heads = num_heads
|
||||
assert self.total_num_heads % tp_size == 0
|
||||
self.num_heads = self.total_num_heads // tp_size
|
||||
self.total_num_kv_heads = num_kv_heads
|
||||
if self.total_num_kv_heads >= tp_size:
|
||||
assert self.total_num_kv_heads % tp_size == 0
|
||||
else:
|
||||
assert tp_size % self.total_num_kv_heads == 0
|
||||
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
|
||||
self.num_heads, self.num_kv_heads = get_attention_head_partition(
|
||||
num_heads, num_kv_heads, tp_size
|
||||
)
|
||||
|
||||
self.head_dim = getattr(
|
||||
config, "head_dim", self.hidden_size // self.total_num_heads
|
||||
@@ -432,38 +430,12 @@ class Qwen3ForCausalLM(TextEncoder):
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
"""Load weights with support for tensor parallelism and weight remapping."""
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
# Strip 'model.' prefix from HuggingFace Qwen3 weights
|
||||
if name.startswith("model."):
|
||||
name = name[6:] # len("model.") == 6
|
||||
|
||||
# Skip rotary embedding weights
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
continue
|
||||
if "rotary_emb.cos_cached" in name or "rotary_emb.sin_cached" in name:
|
||||
continue
|
||||
|
||||
# Handle KV scale remapping
|
||||
if "scale" in name:
|
||||
kv_scale_name: str | None = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if kv_scale_name is None:
|
||||
continue
|
||||
else:
|
||||
name = kv_scale_name
|
||||
|
||||
name = load_stacked_weight(
|
||||
name,
|
||||
loaded_weight,
|
||||
params_dict,
|
||||
self.config.arch_config.stacked_params_mapping,
|
||||
)
|
||||
if name is not None:
|
||||
loaded_params.add(name)
|
||||
|
||||
return loaded_params
|
||||
return load_llm_encoder_weights(
|
||||
weights,
|
||||
dict(self.named_parameters()),
|
||||
self.config.arch_config.stacked_params_mapping,
|
||||
strip_prefix="model.",
|
||||
)
|
||||
|
||||
|
||||
EntryClass = Qwen3ForCausalLM
|
||||
|
||||
@@ -158,9 +158,6 @@ class T5LayerFF(nn.Module):
|
||||
|
||||
# T5 has attn_bias and does not use softmax scaling
|
||||
class T5MultiHeadAttention(nn.Module):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
|
||||
def forward(self, q, k, v, attn_bias=None):
|
||||
b, _, n, c = q.shape
|
||||
attn = torch.einsum("binc,bjnc->bnij", q, k)
|
||||
@@ -509,7 +506,6 @@ class T5Stack(nn.Module):
|
||||
):
|
||||
super().__init__()
|
||||
self.embed_tokens = embed_tokens
|
||||
self.is_umt5 = is_umt5
|
||||
if is_umt5:
|
||||
self.block = nn.ModuleList(
|
||||
[
|
||||
|
||||
@@ -49,7 +49,6 @@ class FlowMatchScheduler(BaseScheduler):
|
||||
self,
|
||||
num_inference_steps=100,
|
||||
denoising_strength=1.0,
|
||||
training=False,
|
||||
shift=None,
|
||||
dynamic_shift_len=None,
|
||||
):
|
||||
@@ -91,17 +90,6 @@ class FlowMatchScheduler(BaseScheduler):
|
||||
if self.train_timesteps is None:
|
||||
self.train_timesteps = self.timesteps
|
||||
self.train_sigmas = self.sigmas
|
||||
if training:
|
||||
x = self.timesteps
|
||||
y = torch.exp(
|
||||
-2 * ((x - num_inference_steps / 2) / num_inference_steps) ** 2
|
||||
)
|
||||
y_shifted = y - y.min()
|
||||
bsmntw_weighing = y_shifted * (num_inference_steps / y_shifted.sum())
|
||||
self.linear_timesteps_weights = bsmntw_weighing
|
||||
self.training = True
|
||||
else:
|
||||
self.training = False
|
||||
|
||||
def scale_model_input(self, sample: torch.Tensor, timestep: int | None = None):
|
||||
return sample
|
||||
@@ -118,14 +106,6 @@ class FlowMatchScheduler(BaseScheduler):
|
||||
prev_sample = sample + model_output * (sigma_ - sigma)
|
||||
return prev_sample
|
||||
|
||||
def return_to_timestep(self, timestep, sample, sample_stablized):
|
||||
if isinstance(timestep, torch.Tensor):
|
||||
timestep = timestep.cpu()
|
||||
timestep_id = torch.argmin((self.timesteps - timestep).abs())
|
||||
sigma = self.sigmas[timestep_id]
|
||||
model_output = (sample - sample_stablized) / sigma
|
||||
return model_output
|
||||
|
||||
def add_noise(self, original_samples, noise, timestep):
|
||||
if isinstance(timestep, torch.Tensor):
|
||||
timestep = timestep.cpu()
|
||||
@@ -134,17 +114,6 @@ class FlowMatchScheduler(BaseScheduler):
|
||||
sample = (1 - sigma) * original_samples + sigma * noise
|
||||
return sample
|
||||
|
||||
def training_target(self, sample, noise, timestep):
|
||||
target = noise - sample
|
||||
return target
|
||||
|
||||
def training_weight(self, timestep):
|
||||
timestep_id = torch.argmin(
|
||||
(self.timesteps - timestep.to(self.timesteps.device)).abs()
|
||||
)
|
||||
weights = self.linear_timesteps_weights[timestep_id]
|
||||
return weights
|
||||
|
||||
def calculate_shift(
|
||||
self,
|
||||
image_seq_len,
|
||||
@@ -268,9 +237,9 @@ class FlowMatchPairScheduler(FlowMatchScheduler):
|
||||
|
||||
self.set_pair_postprocess(_quadratic_perp_bulge_swap)
|
||||
return
|
||||
if name == "v2a_sequential":
|
||||
if name in ("v2a_sequential", "a2v_sequential"):
|
||||
|
||||
def _v2a(pairs: torch.Tensor):
|
||||
def _sequential(pairs: torch.Tensor):
|
||||
if (
|
||||
not isinstance(pairs, torch.Tensor)
|
||||
or pairs.ndim != 2
|
||||
@@ -281,8 +250,12 @@ class FlowMatchPairScheduler(FlowMatchScheduler):
|
||||
base = pairs[:, 0]
|
||||
seq_half = base[::2]
|
||||
m = int(seq_half.shape[0])
|
||||
col0 = torch.cat([seq_half, seq_half[-1:].repeat(m)], dim=0)[:N]
|
||||
col1 = torch.cat([seq_half[0:1].repeat(m), seq_half], dim=0)[:N]
|
||||
if name == "v2a_sequential":
|
||||
col0 = torch.cat([seq_half, seq_half[-1:].repeat(m)], dim=0)[:N]
|
||||
col1 = torch.cat([seq_half[0:1].repeat(m), seq_half], dim=0)[:N]
|
||||
else:
|
||||
col0 = torch.cat([seq_half[0:1].repeat(m), seq_half], dim=0)[:N]
|
||||
col1 = torch.cat([seq_half, seq_half[-1:].repeat(m)], dim=0)[:N]
|
||||
return torch.stack(
|
||||
[
|
||||
col0.to(dtype=pairs.dtype, device=pairs.device),
|
||||
@@ -291,60 +264,25 @@ class FlowMatchPairScheduler(FlowMatchScheduler):
|
||||
dim=1,
|
||||
)
|
||||
|
||||
self.set_pair_postprocess(_v2a)
|
||||
self.set_pair_postprocess(_sequential)
|
||||
return
|
||||
if name == "a2v_sequential":
|
||||
if name in ("v2a", "a2v"):
|
||||
|
||||
def _a2v(pairs: torch.Tensor):
|
||||
def _single_modality(pairs: torch.Tensor):
|
||||
if (
|
||||
not isinstance(pairs, torch.Tensor)
|
||||
or pairs.ndim != 2
|
||||
or pairs.shape[1] != 2
|
||||
):
|
||||
raise ValueError("pairs must be a torch.Tensor of shape [N, 2]")
|
||||
N = pairs.shape[0]
|
||||
base = pairs[:, 0]
|
||||
seq_half = base[::2]
|
||||
m = int(seq_half.shape[0])
|
||||
col0 = torch.cat([seq_half[0:1].repeat(m), seq_half], dim=0)[:N]
|
||||
col1 = torch.cat([seq_half, seq_half[-1:].repeat(m)], dim=0)[:N]
|
||||
return torch.stack(
|
||||
[
|
||||
col0.to(dtype=pairs.dtype, device=pairs.device),
|
||||
col1.to(dtype=pairs.dtype, device=pairs.device),
|
||||
],
|
||||
dim=1,
|
||||
)
|
||||
if name == "v2a":
|
||||
zeros = torch.zeros_like(pairs[:, 0])
|
||||
return torch.stack([zeros, pairs[:, 1]], dim=1)
|
||||
else:
|
||||
zeros = torch.zeros_like(pairs[:, 1])
|
||||
return torch.stack([pairs[:, 0], zeros], dim=1)
|
||||
|
||||
self.set_pair_postprocess(_a2v)
|
||||
return
|
||||
if name == "v2a":
|
||||
|
||||
def _v2a_classic(pairs: torch.Tensor):
|
||||
if (
|
||||
not isinstance(pairs, torch.Tensor)
|
||||
or pairs.ndim != 2
|
||||
or pairs.shape[1] != 2
|
||||
):
|
||||
raise ValueError("pairs must be a torch.Tensor of shape [N, 2]")
|
||||
zeros = torch.zeros_like(pairs[:, 0])
|
||||
return torch.stack([zeros, pairs[:, 1]], dim=1)
|
||||
|
||||
self.set_pair_postprocess(_v2a_classic)
|
||||
return
|
||||
if name == "a2v":
|
||||
|
||||
def _a2v_classic(pairs: torch.Tensor):
|
||||
if (
|
||||
not isinstance(pairs, torch.Tensor)
|
||||
or pairs.ndim != 2
|
||||
or pairs.shape[1] != 2
|
||||
):
|
||||
raise ValueError("pairs must be a torch.Tensor of shape [N, 2]")
|
||||
zeros = torch.zeros_like(pairs[:, 1])
|
||||
return torch.stack([pairs[:, 0], zeros], dim=1)
|
||||
|
||||
self.set_pair_postprocess(_a2v_classic)
|
||||
self.set_pair_postprocess(_single_modality)
|
||||
return
|
||||
if name == "dual_sigma_shift":
|
||||
visual_shift = float(kwargs.get("visual_shift", self.shift))
|
||||
|
||||
@@ -19,21 +19,7 @@ import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
if "sinc" in dir(torch):
|
||||
sinc = torch.sinc
|
||||
else:
|
||||
# This code is adopted from adefossez's julius.core.sinc under the MIT License
|
||||
# https://adefossez.github.io/julius/julius/core.html
|
||||
def sinc(x: torch.Tensor):
|
||||
"""
|
||||
Implementation of sinc, i.e. sin(pi * x) / (pi * x)
|
||||
__Warning__: Different to julius.sinc, the input is multiplied by `pi`!
|
||||
"""
|
||||
return torch.where(
|
||||
x == 0,
|
||||
torch.tensor(1.0, device=x.device, dtype=x.dtype),
|
||||
torch.sin(math.pi * x) / math.pi / x,
|
||||
)
|
||||
sinc = torch.sinc
|
||||
|
||||
|
||||
# This code is adopted from adefossez's julius.lowpass.LowPassFilters under the MIT License
|
||||
@@ -175,8 +161,6 @@ class Activation1d(nn.Module):
|
||||
down_kernel_size: int = 12,
|
||||
):
|
||||
super().__init__()
|
||||
self.up_ratio = up_ratio
|
||||
self.down_ratio = down_ratio
|
||||
self.act = activation
|
||||
self.upsample = UpSample1d(up_ratio, up_kernel_size)
|
||||
self.downsample = DownSample1d(down_ratio, down_kernel_size)
|
||||
|
||||
@@ -951,18 +951,6 @@ class AutoencoderKLQwenImage(ParallelTiledVAE):
|
||||
self.tile_sample_stride_height = 192
|
||||
self.tile_sample_stride_width = 192
|
||||
|
||||
# Precompute and cache conv counts for encoder and decoder for clear_cache speedup
|
||||
self._cached_conv_counts = {
|
||||
"decoder": sum(
|
||||
isinstance(m, (QwenImageCausalConv3d, SpatialParallelCausalConv3d))
|
||||
for m in self.decoder.modules()
|
||||
)
|
||||
if self.decoder is not None
|
||||
else 0,
|
||||
"encoder": sum(isinstance(m, QwenImageCausalConv3d) for m in self.encoder.modules())
|
||||
if self.encoder is not None
|
||||
else 0,
|
||||
}
|
||||
cuda_device = get_local_torch_device()
|
||||
dtype = torch.get_default_dtype()
|
||||
latent_channels = config.arch_config.z_dim
|
||||
|
||||
@@ -7,7 +7,6 @@ from functools import lru_cache
|
||||
from math import isqrt, prod
|
||||
from typing import Optional, cast
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
from diffusers.models.autoencoders.vae import DiagonalGaussianDistribution
|
||||
@@ -230,6 +229,14 @@ class ParallelTiledVAE(ABC, nn.Module, LayerwiseOffloadableModuleMixin):
|
||||
) + b[:, :, x, :, :] * (x / blend_extent)
|
||||
return b
|
||||
|
||||
def _spatial_tile_geometry(self) -> tuple[int, int, int, int]:
|
||||
return (
|
||||
self.tile_sample_min_height // self.spatial_compression_ratio,
|
||||
self.tile_sample_min_width // self.spatial_compression_ratio,
|
||||
self.tile_sample_stride_height // self.spatial_compression_ratio,
|
||||
self.tile_sample_stride_width // self.spatial_compression_ratio,
|
||||
)
|
||||
|
||||
def spatial_tiled_encode(self, x: torch.Tensor) -> torch.Tensor:
|
||||
r"""Encode a batch of images using a tiled encoder.
|
||||
|
||||
@@ -244,18 +251,12 @@ class ParallelTiledVAE(ABC, nn.Module, LayerwiseOffloadableModuleMixin):
|
||||
# latent_height = height // self.spatial_compression_ratio
|
||||
# latent_width = width // self.spatial_compression_ratio
|
||||
|
||||
tile_latent_min_height = (
|
||||
self.tile_sample_min_height // self.spatial_compression_ratio
|
||||
)
|
||||
tile_latent_min_width = (
|
||||
self.tile_sample_min_width // self.spatial_compression_ratio
|
||||
)
|
||||
tile_latent_stride_height = (
|
||||
self.tile_sample_stride_height // self.spatial_compression_ratio
|
||||
)
|
||||
tile_latent_stride_width = (
|
||||
self.tile_sample_stride_width // self.spatial_compression_ratio
|
||||
)
|
||||
(
|
||||
tile_latent_min_height,
|
||||
tile_latent_min_width,
|
||||
tile_latent_stride_height,
|
||||
tile_latent_stride_width,
|
||||
) = self._spatial_tile_geometry()
|
||||
|
||||
blend_height = tile_latent_min_height - tile_latent_stride_height
|
||||
blend_width = tile_latent_min_width - tile_latent_stride_width
|
||||
@@ -293,21 +294,15 @@ class ParallelTiledVAE(ABC, nn.Module, LayerwiseOffloadableModuleMixin):
|
||||
world_size, rank = sp_group.world_size, sp_group.rank_in_group
|
||||
_, _, T, H, W = z.shape
|
||||
|
||||
tile_latent_min_height = (
|
||||
self.tile_sample_min_height // self.spatial_compression_ratio
|
||||
)
|
||||
tile_latent_min_width = (
|
||||
self.tile_sample_min_width // self.spatial_compression_ratio
|
||||
)
|
||||
(
|
||||
tile_latent_min_height,
|
||||
tile_latent_min_width,
|
||||
tile_latent_stride_height,
|
||||
tile_latent_stride_width,
|
||||
) = self._spatial_tile_geometry()
|
||||
tile_latent_min_num_frames = (
|
||||
self.tile_sample_min_num_frames // self.temporal_compression_ratio
|
||||
)
|
||||
tile_latent_stride_height = (
|
||||
self.tile_sample_stride_height // self.spatial_compression_ratio
|
||||
)
|
||||
tile_latent_stride_width = (
|
||||
self.tile_sample_stride_width // self.spatial_compression_ratio
|
||||
)
|
||||
tile_latent_stride_num_frames = (
|
||||
self.tile_sample_stride_num_frames // self.temporal_compression_ratio
|
||||
)
|
||||
@@ -436,18 +431,12 @@ class ParallelTiledVAE(ABC, nn.Module, LayerwiseOffloadableModuleMixin):
|
||||
if world_size <= 1:
|
||||
return self._decode(z)
|
||||
|
||||
tile_latent_min_height = (
|
||||
self.tile_sample_min_height // self.spatial_compression_ratio
|
||||
)
|
||||
tile_latent_min_width = (
|
||||
self.tile_sample_min_width // self.spatial_compression_ratio
|
||||
)
|
||||
tile_latent_stride_height = (
|
||||
self.tile_sample_stride_height // self.spatial_compression_ratio
|
||||
)
|
||||
tile_latent_stride_width = (
|
||||
self.tile_sample_stride_width // self.spatial_compression_ratio
|
||||
)
|
||||
(
|
||||
tile_latent_min_height,
|
||||
tile_latent_min_width,
|
||||
tile_latent_stride_height,
|
||||
tile_latent_stride_width,
|
||||
) = self._spatial_tile_geometry()
|
||||
overlap_h = max(0, tile_latent_min_height - tile_latent_stride_height)
|
||||
overlap_w = max(0, tile_latent_min_width - tile_latent_stride_width)
|
||||
halo_h = overlap_h // 2
|
||||
@@ -574,18 +563,12 @@ class ParallelTiledVAE(ABC, nn.Module, LayerwiseOffloadableModuleMixin):
|
||||
# sample_height = height * self.spatial_compression_ratio
|
||||
# sample_width = width * self.spatial_compression_ratio
|
||||
|
||||
tile_latent_min_height = (
|
||||
self.tile_sample_min_height // self.spatial_compression_ratio
|
||||
)
|
||||
tile_latent_min_width = (
|
||||
self.tile_sample_min_width // self.spatial_compression_ratio
|
||||
)
|
||||
tile_latent_stride_height = (
|
||||
self.tile_sample_stride_height // self.spatial_compression_ratio
|
||||
)
|
||||
tile_latent_stride_width = (
|
||||
self.tile_sample_stride_width // self.spatial_compression_ratio
|
||||
)
|
||||
(
|
||||
tile_latent_min_height,
|
||||
tile_latent_min_width,
|
||||
tile_latent_stride_height,
|
||||
tile_latent_stride_width,
|
||||
) = self._spatial_tile_geometry()
|
||||
|
||||
blend_height = self.tile_sample_min_height - self.tile_sample_stride_height
|
||||
blend_width = self.tile_sample_min_width - self.tile_sample_stride_width
|
||||
@@ -809,16 +792,5 @@ class DiagonalGaussianDistribution:
|
||||
dim=dims,
|
||||
)
|
||||
|
||||
def nll(
|
||||
self, sample: torch.Tensor, dims: tuple[int, ...] = (1, 2, 3)
|
||||
) -> torch.Tensor:
|
||||
if self.deterministic:
|
||||
return torch.Tensor([0.0])
|
||||
logtwopi = np.log(2.0 * np.pi)
|
||||
return 0.5 * torch.sum(
|
||||
logtwopi + self.logvar + torch.pow(sample - self.mean, 2) / self.var,
|
||||
dim=dims,
|
||||
)
|
||||
|
||||
def mode(self) -> torch.Tensor:
|
||||
return self.mean
|
||||
|
||||
@@ -193,12 +193,6 @@ class Cosmos3AVAEAudioTokenizer(nn.Module, LayerwiseOffloadableModuleMixin):
|
||||
def temporal_compression_factor(self) -> int:
|
||||
return self.hop_size
|
||||
|
||||
def get_latent_num_samples(self, num_audio_samples: int) -> int:
|
||||
return int(num_audio_samples) // self.hop_size
|
||||
|
||||
def get_audio_num_samples(self, num_latent_samples: int) -> int:
|
||||
return int(num_latent_samples) * self.hop_size
|
||||
|
||||
def _denormalize_latent(self, latent: torch.Tensor) -> torch.Tensor:
|
||||
if self.normalization_type == "tanh":
|
||||
in_dtype = latent.dtype
|
||||
|
||||
@@ -370,7 +370,6 @@ class Encoder(nn.Module):
|
||||
|
||||
# Wrap black into nn.Sequential
|
||||
self.block = nn.Sequential(*self.block)
|
||||
self.enc_dim = d_model
|
||||
|
||||
def forward(self, x):
|
||||
return self.block(x)
|
||||
|
||||
@@ -209,7 +209,6 @@ class QKVMultiheadCrossAttention(nn.Module):
|
||||
):
|
||||
super().__init__()
|
||||
self.heads = heads
|
||||
self.n_data = n_data
|
||||
self.q_norm = (
|
||||
norm_layer(width // heads, elementwise_affine=True, eps=1e-6)
|
||||
if qk_norm
|
||||
@@ -255,7 +254,6 @@ class MultiheadCrossAttention(nn.Module):
|
||||
kv_cache: bool = False,
|
||||
):
|
||||
super().__init__()
|
||||
self.n_data = n_data
|
||||
self.width = width
|
||||
self.heads = heads
|
||||
self.data_width = width if data_width is None else data_width
|
||||
@@ -338,7 +336,6 @@ class QKVMultiheadAttention(nn.Module):
|
||||
):
|
||||
super().__init__()
|
||||
self.heads = heads
|
||||
self.n_ctx = n_ctx
|
||||
self.q_norm = (
|
||||
norm_layer(width // heads, elementwise_affine=True, eps=1e-6)
|
||||
if qk_norm
|
||||
@@ -381,7 +378,6 @@ class MultiheadAttention(nn.Module):
|
||||
drop_path_rate: float = 0.0,
|
||||
):
|
||||
super().__init__()
|
||||
self.n_ctx = n_ctx
|
||||
self.width = width
|
||||
self.heads = heads
|
||||
self.c_qkv = nn.Linear(width, width * 3, bias=qkv_bias)
|
||||
@@ -450,7 +446,6 @@ class Transformer(nn.Module):
|
||||
drop_path_rate: float = 0.0,
|
||||
):
|
||||
super().__init__()
|
||||
self.n_ctx = n_ctx
|
||||
self.width = width
|
||||
self.layers = layers
|
||||
self.resblocks = nn.ModuleList(
|
||||
@@ -512,7 +507,6 @@ class CrossAttentionDecoder(nn.Module):
|
||||
if self.enable_ln_post:
|
||||
self.ln_post = nn.LayerNorm(width)
|
||||
self.output_proj = nn.Linear(width, out_channels)
|
||||
self.label_type = label_type
|
||||
|
||||
def set_cross_attention_processor(self, processor):
|
||||
self.cross_attn_decoder.attn.attention.attn_processor = processor
|
||||
|
||||
@@ -131,7 +131,6 @@ class LTX23VideoConditionEncoder(
|
||||
latent_log_var = str(vae_config.get("latent_log_var", "uniform"))
|
||||
|
||||
self.patch_size = patch_size
|
||||
self.latency_channels = latent_channels
|
||||
self.latent_log_var = latent_log_var
|
||||
self.per_channel_statistics = LTX23PerChannelStatistics(latent_channels)
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user