[diffusion] refactor: share sigma schedules and causal cross-attention (#42848)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
@@ -330,6 +330,42 @@ class CausalWanTransformerBlock(nn.Module):
|
||||
|
||||
self.scale_shift_table = nn.Parameter(torch.randn(1, 6, dim) / dim**0.5)
|
||||
|
||||
def _cross_attn_with_cache(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
encoder_hidden_states: torch.Tensor,
|
||||
crossattn_cache: CrossAttentionKVCache | None,
|
||||
) -> torch.Tensor:
|
||||
attn2 = self.attn2
|
||||
q, _ = attn2.to_q(hidden_states)
|
||||
if attn2.tp_rmsnorm:
|
||||
q = tensor_parallel_rms_norm(q, attn2.norm_q)
|
||||
else:
|
||||
q = attn2.norm_q(q)
|
||||
q = q.unflatten(2, (attn2.local_num_heads, attn2.head_dim))
|
||||
|
||||
if crossattn_cache is not None and crossattn_cache.is_init:
|
||||
k = crossattn_cache.k
|
||||
v = crossattn_cache.v
|
||||
else:
|
||||
k, _ = attn2.to_k(encoder_hidden_states)
|
||||
if attn2.tp_rmsnorm:
|
||||
k = tensor_parallel_rms_norm(k, attn2.norm_k)
|
||||
else:
|
||||
k = attn2.norm_k(k)
|
||||
k = k.unflatten(2, (attn2.local_num_heads, attn2.head_dim))
|
||||
|
||||
v, _ = attn2.to_v(encoder_hidden_states)
|
||||
v = v.unflatten(2, (attn2.local_num_heads, attn2.head_dim))
|
||||
|
||||
if crossattn_cache is not None:
|
||||
crossattn_cache.store(k, v)
|
||||
|
||||
hidden_states = attn2.attn(q, k, v)
|
||||
hidden_states = hidden_states.flatten(2)
|
||||
hidden_states, _ = attn2.to_out(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
|
||||
@@ -1146,42 +1146,6 @@ class CausalLingBotWorldTransformerBlock(CausalWanTransformerBlock):
|
||||
value, _ = self.to_v(hidden_states)
|
||||
return query, key, value
|
||||
|
||||
def _cross_attn_with_cache(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
encoder_hidden_states: torch.Tensor,
|
||||
crossattn_cache: CrossAttentionKVCache | None,
|
||||
) -> torch.Tensor:
|
||||
attn2 = self.attn2
|
||||
q, _ = attn2.to_q(hidden_states)
|
||||
if attn2.tp_rmsnorm:
|
||||
q = tensor_parallel_rms_norm(q, attn2.norm_q)
|
||||
else:
|
||||
q = attn2.norm_q(q)
|
||||
q = q.unflatten(2, (attn2.local_num_heads, attn2.head_dim))
|
||||
|
||||
if crossattn_cache is not None and crossattn_cache.is_init:
|
||||
k = crossattn_cache.k
|
||||
v = crossattn_cache.v
|
||||
else:
|
||||
k, _ = attn2.to_k(encoder_hidden_states)
|
||||
if attn2.tp_rmsnorm:
|
||||
k = tensor_parallel_rms_norm(k, attn2.norm_k)
|
||||
else:
|
||||
k = attn2.norm_k(k)
|
||||
k = k.unflatten(2, (attn2.local_num_heads, attn2.head_dim))
|
||||
|
||||
v, _ = attn2.to_v(encoder_hidden_states)
|
||||
v = v.unflatten(2, (attn2.local_num_heads, attn2.head_dim))
|
||||
|
||||
if crossattn_cache is not None:
|
||||
crossattn_cache.store(k, v)
|
||||
|
||||
hidden_states = attn2.attn(q, k, v)
|
||||
hidden_states = hidden_states.flatten(2)
|
||||
hidden_states, _ = attn2.to_out(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
def _cam_conditioner_scale_shift(
|
||||
self,
|
||||
c2ws_plucker_emb: torch.Tensor | None,
|
||||
|
||||
@@ -13,7 +13,6 @@ from sglang.multimodal_gen.runtime.layers.kvcache.causal_attention_cache import
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.layernorm import (
|
||||
LayerNormScaleShift,
|
||||
tensor_parallel_rms_norm,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config import (
|
||||
QuantizationConfig,
|
||||
@@ -34,42 +33,6 @@ class LongLive2CausalWanTransformerBlock(CausalWanTransformerBlock):
|
||||
dtype=torch.float32,
|
||||
)
|
||||
|
||||
def _cross_attn_with_cache(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
encoder_hidden_states: torch.Tensor,
|
||||
crossattn_cache: CrossAttentionKVCache | None,
|
||||
) -> torch.Tensor:
|
||||
attn2 = self.attn2
|
||||
q, _ = attn2.to_q(hidden_states)
|
||||
if attn2.tp_rmsnorm:
|
||||
q = tensor_parallel_rms_norm(q, attn2.norm_q)
|
||||
else:
|
||||
q = attn2.norm_q(q)
|
||||
q = q.unflatten(2, (attn2.local_num_heads, attn2.head_dim))
|
||||
|
||||
if crossattn_cache is not None and crossattn_cache.is_init:
|
||||
k = crossattn_cache.k
|
||||
v = crossattn_cache.v
|
||||
else:
|
||||
k, _ = attn2.to_k(encoder_hidden_states)
|
||||
if attn2.tp_rmsnorm:
|
||||
k = tensor_parallel_rms_norm(k, attn2.norm_k)
|
||||
else:
|
||||
k = attn2.norm_k(k)
|
||||
k = k.unflatten(2, (attn2.local_num_heads, attn2.head_dim))
|
||||
|
||||
v, _ = attn2.to_v(encoder_hidden_states)
|
||||
v = v.unflatten(2, (attn2.local_num_heads, attn2.head_dim))
|
||||
|
||||
if crossattn_cache is not None:
|
||||
crossattn_cache.store(k, v)
|
||||
|
||||
hidden_states = attn2.attn(q, k, v)
|
||||
hidden_states = hidden_states.flatten(2)
|
||||
hidden_states, _ = attn2.to_out(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
|
||||
+7
-98
@@ -25,13 +25,15 @@ from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import scipy.stats
|
||||
import torch
|
||||
from diffusers.configuration_utils import ConfigMixin, register_to_config
|
||||
from diffusers.schedulers.scheduling_utils import SchedulerMixin
|
||||
from diffusers.utils import BaseOutput
|
||||
|
||||
from sglang.multimodal_gen.runtime.models.schedulers.base import BaseScheduler
|
||||
from sglang.multimodal_gen.runtime.models.schedulers.sigma_schedule import (
|
||||
SigmaScheduleMixin,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.post_training.scheduler_rl_mixin import (
|
||||
SchedulerRLMixin,
|
||||
)
|
||||
@@ -55,7 +57,7 @@ class FlowMatchEulerDiscreteSchedulerOutput(BaseOutput):
|
||||
|
||||
|
||||
class FlowMatchEulerDiscreteScheduler(
|
||||
SchedulerMixin, ConfigMixin, BaseScheduler, SchedulerRLMixin
|
||||
SchedulerMixin, ConfigMixin, BaseScheduler, SchedulerRLMixin, SigmaScheduleMixin
|
||||
):
|
||||
"""
|
||||
Euler scheduler.
|
||||
@@ -369,22 +371,19 @@ class FlowMatchEulerDiscreteScheduler(
|
||||
# 4. If required, convert sigmas to one of karras, exponential, or beta sigma schedules
|
||||
if self.config.use_karras_sigmas:
|
||||
sigmas_tensor = torch.from_numpy(sigmas_array).to(dtype=torch.float32)
|
||||
sigmas_tensor = self._convert_to_karras(
|
||||
sigmas_array = self._convert_to_karras(
|
||||
in_sigmas=sigmas_tensor, num_inference_steps=num_inference_steps
|
||||
)
|
||||
sigmas_array = sigmas_tensor.numpy()
|
||||
elif self.config.use_exponential_sigmas:
|
||||
sigmas_tensor = torch.from_numpy(sigmas_array).to(dtype=torch.float32)
|
||||
sigmas_tensor = self._convert_to_exponential(
|
||||
sigmas_array = self._convert_to_exponential(
|
||||
in_sigmas=sigmas_tensor, num_inference_steps=num_inference_steps
|
||||
)
|
||||
sigmas_array = sigmas_tensor.numpy()
|
||||
elif self.config.use_beta_sigmas:
|
||||
sigmas_tensor = torch.from_numpy(sigmas_array).to(dtype=torch.float32)
|
||||
sigmas_tensor = self._convert_to_beta(
|
||||
sigmas_array = self._convert_to_beta(
|
||||
in_sigmas=sigmas_tensor, num_inference_steps=num_inference_steps
|
||||
)
|
||||
sigmas_array = sigmas_tensor.numpy()
|
||||
|
||||
# 5. Convert sigmas and timesteps to tensors and move to specified device
|
||||
sigmas_tensor = torch.from_numpy(sigmas_array).to(
|
||||
@@ -555,96 +554,6 @@ class FlowMatchEulerDiscreteScheduler(
|
||||
|
||||
return FlowMatchEulerDiscreteSchedulerOutput(prev_sample=prev_sample)
|
||||
|
||||
# Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_karras
|
||||
def _convert_to_karras(
|
||||
self, in_sigmas: torch.Tensor, num_inference_steps: int
|
||||
) -> torch.Tensor:
|
||||
"""Constructs the noise schedule of Karras et al. (2022)."""
|
||||
|
||||
# Hack to make sure that other schedulers which copy this function don't break
|
||||
# TODO: Add this logic to the other schedulers
|
||||
if hasattr(self.config, "sigma_min"):
|
||||
sigma_min = self.config.sigma_min
|
||||
else:
|
||||
sigma_min = None
|
||||
|
||||
if hasattr(self.config, "sigma_max"):
|
||||
sigma_max = self.config.sigma_max
|
||||
else:
|
||||
sigma_max = None
|
||||
|
||||
sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item()
|
||||
sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item()
|
||||
|
||||
rho = 7.0 # 7.0 is the value used in the paper
|
||||
ramp = np.linspace(0, 1, num_inference_steps)
|
||||
min_inv_rho = sigma_min ** (1 / rho)
|
||||
max_inv_rho = sigma_max ** (1 / rho)
|
||||
sigmas = (max_inv_rho + ramp * (min_inv_rho - max_inv_rho)) ** rho
|
||||
return sigmas
|
||||
|
||||
# Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_exponential
|
||||
def _convert_to_exponential(
|
||||
self, in_sigmas: torch.Tensor, num_inference_steps: int
|
||||
) -> torch.Tensor:
|
||||
"""Constructs an exponential noise schedule."""
|
||||
|
||||
# Hack to make sure that other schedulers which copy this function don't break
|
||||
# TODO: Add this logic to the other schedulers
|
||||
if hasattr(self.config, "sigma_min"):
|
||||
sigma_min = self.config.sigma_min
|
||||
else:
|
||||
sigma_min = None
|
||||
|
||||
if hasattr(self.config, "sigma_max"):
|
||||
sigma_max = self.config.sigma_max
|
||||
else:
|
||||
sigma_max = None
|
||||
|
||||
sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item()
|
||||
sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item()
|
||||
|
||||
sigmas = np.exp(
|
||||
np.linspace(math.log(sigma_max), math.log(sigma_min), num_inference_steps)
|
||||
)
|
||||
return sigmas
|
||||
|
||||
# Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_beta
|
||||
def _convert_to_beta(
|
||||
self,
|
||||
in_sigmas: torch.Tensor,
|
||||
num_inference_steps: int,
|
||||
alpha: float = 0.6,
|
||||
beta: float = 0.6,
|
||||
) -> torch.Tensor:
|
||||
"""From "Beta Sampling is All You Need" [arXiv:2407.12173] (Lee et. al, 2024)"""
|
||||
|
||||
# Hack to make sure that other schedulers which copy this function don't break
|
||||
# TODO: Add this logic to the other schedulers
|
||||
if hasattr(self.config, "sigma_min"):
|
||||
sigma_min = self.config.sigma_min
|
||||
else:
|
||||
sigma_min = None
|
||||
|
||||
if hasattr(self.config, "sigma_max"):
|
||||
sigma_max = self.config.sigma_max
|
||||
else:
|
||||
sigma_max = None
|
||||
|
||||
sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item()
|
||||
sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item()
|
||||
|
||||
sigmas = np.array(
|
||||
[
|
||||
sigma_min + (ppf * (sigma_max - sigma_min))
|
||||
for ppf in [
|
||||
scipy.stats.beta.ppf(timestep, alpha, beta)
|
||||
for timestep in 1 - np.linspace(0, 1, num_inference_steps)
|
||||
]
|
||||
]
|
||||
)
|
||||
return sigmas
|
||||
|
||||
def _time_shift_exponential(
|
||||
self, mu: float, sigma: float, t: torch.Tensor | np.ndarray
|
||||
) -> torch.Tensor | np.ndarray:
|
||||
|
||||
+6
-94
@@ -35,9 +35,9 @@ from diffusers.schedulers.scheduling_utils import (
|
||||
from diffusers.utils import deprecate, is_scipy_available
|
||||
|
||||
from sglang.multimodal_gen.runtime.models.schedulers.base import BaseScheduler
|
||||
|
||||
if is_scipy_available():
|
||||
import scipy.stats
|
||||
from sglang.multimodal_gen.runtime.models.schedulers.sigma_schedule import (
|
||||
SigmaScheduleMixin,
|
||||
)
|
||||
|
||||
|
||||
# Copied from diffusers.schedulers.scheduling_ddpm.betas_for_alpha_bar
|
||||
@@ -122,7 +122,9 @@ def rescale_zero_terminal_snr(betas):
|
||||
return betas
|
||||
|
||||
|
||||
class UniPCMultistepScheduler(SchedulerMixin, ConfigMixin, BaseScheduler):
|
||||
class UniPCMultistepScheduler(
|
||||
SchedulerMixin, ConfigMixin, BaseScheduler, SigmaScheduleMixin
|
||||
):
|
||||
"""
|
||||
`UniPCMultistepScheduler` is a training-free framework designed for the fast sampling of diffusion models.
|
||||
|
||||
@@ -614,96 +616,6 @@ class UniPCMultistepScheduler(SchedulerMixin, ConfigMixin, BaseScheduler):
|
||||
|
||||
return alpha_t, sigma_t
|
||||
|
||||
# Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_karras
|
||||
def _convert_to_karras(
|
||||
self, in_sigmas: torch.Tensor, num_inference_steps
|
||||
) -> torch.Tensor:
|
||||
"""Constructs the noise schedule of Karras et al. (2022)."""
|
||||
|
||||
# Hack to make sure that other schedulers which copy this function don't break
|
||||
# TODO: Add this logic to the other schedulers
|
||||
if hasattr(self.config, "sigma_min"):
|
||||
sigma_min = self.config.sigma_min
|
||||
else:
|
||||
sigma_min = None
|
||||
|
||||
if hasattr(self.config, "sigma_max"):
|
||||
sigma_max = self.config.sigma_max
|
||||
else:
|
||||
sigma_max = None
|
||||
|
||||
sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item()
|
||||
sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item()
|
||||
|
||||
rho = 7.0 # 7.0 is the value used in the paper
|
||||
ramp = np.linspace(0, 1, num_inference_steps)
|
||||
min_inv_rho = sigma_min ** (1 / rho)
|
||||
max_inv_rho = sigma_max ** (1 / rho)
|
||||
sigmas = (max_inv_rho + ramp * (min_inv_rho - max_inv_rho)) ** rho
|
||||
return sigmas
|
||||
|
||||
# Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_exponential
|
||||
def _convert_to_exponential(
|
||||
self, in_sigmas: torch.Tensor, num_inference_steps: int
|
||||
) -> torch.Tensor:
|
||||
"""Constructs an exponential noise schedule."""
|
||||
|
||||
# Hack to make sure that other schedulers which copy this function don't break
|
||||
# TODO: Add this logic to the other schedulers
|
||||
if hasattr(self.config, "sigma_min"):
|
||||
sigma_min = self.config.sigma_min
|
||||
else:
|
||||
sigma_min = None
|
||||
|
||||
if hasattr(self.config, "sigma_max"):
|
||||
sigma_max = self.config.sigma_max
|
||||
else:
|
||||
sigma_max = None
|
||||
|
||||
sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item()
|
||||
sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item()
|
||||
|
||||
sigmas = np.exp(
|
||||
np.linspace(math.log(sigma_max), math.log(sigma_min), num_inference_steps)
|
||||
)
|
||||
return sigmas
|
||||
|
||||
# Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_beta
|
||||
def _convert_to_beta(
|
||||
self,
|
||||
in_sigmas: torch.Tensor,
|
||||
num_inference_steps: int,
|
||||
alpha: float = 0.6,
|
||||
beta: float = 0.6,
|
||||
) -> torch.Tensor:
|
||||
"""From "Beta Sampling is All You Need" [arXiv:2407.12173] (Lee et. al, 2024)"""
|
||||
|
||||
# Hack to make sure that other schedulers which copy this function don't break
|
||||
# TODO: Add this logic to the other schedulers
|
||||
if hasattr(self.config, "sigma_min"):
|
||||
sigma_min = self.config.sigma_min
|
||||
else:
|
||||
sigma_min = None
|
||||
|
||||
if hasattr(self.config, "sigma_max"):
|
||||
sigma_max = self.config.sigma_max
|
||||
else:
|
||||
sigma_max = None
|
||||
|
||||
sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item()
|
||||
sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item()
|
||||
|
||||
sigmas = np.array(
|
||||
[
|
||||
sigma_min + (ppf * (sigma_max - sigma_min))
|
||||
for ppf in [
|
||||
scipy.stats.beta.ppf(timestep, alpha, beta)
|
||||
for timestep in 1 - np.linspace(0, 1, num_inference_steps)
|
||||
]
|
||||
]
|
||||
)
|
||||
return sigmas
|
||||
|
||||
def convert_model_output(
|
||||
self,
|
||||
model_output: torch.Tensor,
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2024 Stability AI, Katherine Crowson and The HuggingFace Team.
|
||||
# Copyright 2025 TSAIL Team and The HuggingFace Team.
|
||||
# Adapted from diffusers.schedulers.scheduling_euler_discrete.
|
||||
|
||||
import math
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from diffusers.utils import is_scipy_available
|
||||
|
||||
if is_scipy_available():
|
||||
import scipy.stats
|
||||
|
||||
|
||||
class SigmaScheduleMixin:
|
||||
"""Alternative sigma schedules for schedulers with a Diffusers config."""
|
||||
|
||||
def _sigma_bounds(self, in_sigmas):
|
||||
sigma_min = self.config.get("sigma_min")
|
||||
sigma_max = self.config.get("sigma_max")
|
||||
sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item()
|
||||
sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item()
|
||||
return sigma_min, sigma_max
|
||||
|
||||
def _convert_to_karras(
|
||||
self, in_sigmas: torch.Tensor, num_inference_steps: int
|
||||
) -> np.ndarray:
|
||||
"""Construct the Karras et al. (2022) noise schedule."""
|
||||
sigma_min, sigma_max = self._sigma_bounds(in_sigmas)
|
||||
rho = 7.0
|
||||
ramp = np.linspace(0, 1, num_inference_steps)
|
||||
min_inv_rho = sigma_min ** (1 / rho)
|
||||
max_inv_rho = sigma_max ** (1 / rho)
|
||||
return (max_inv_rho + ramp * (min_inv_rho - max_inv_rho)) ** rho
|
||||
|
||||
def _convert_to_exponential(
|
||||
self, in_sigmas: torch.Tensor, num_inference_steps: int
|
||||
) -> np.ndarray:
|
||||
"""Construct an exponential noise schedule."""
|
||||
sigma_min, sigma_max = self._sigma_bounds(in_sigmas)
|
||||
return np.exp(
|
||||
np.linspace(math.log(sigma_max), math.log(sigma_min), num_inference_steps)
|
||||
)
|
||||
|
||||
def _convert_to_beta(
|
||||
self,
|
||||
in_sigmas: torch.Tensor,
|
||||
num_inference_steps: int,
|
||||
alpha: float = 0.6,
|
||||
beta: float = 0.6,
|
||||
) -> np.ndarray:
|
||||
"""Construct the Beta Sampling (Lee et al., 2024) noise schedule."""
|
||||
sigma_min, sigma_max = self._sigma_bounds(in_sigmas)
|
||||
return np.array(
|
||||
[
|
||||
sigma_min + (ppf * (sigma_max - sigma_min))
|
||||
for ppf in [
|
||||
scipy.stats.beta.ppf(timestep, alpha, beta)
|
||||
for timestep in 1 - np.linspace(0, 1, num_inference_steps)
|
||||
]
|
||||
]
|
||||
)
|
||||
@@ -1,18 +1,24 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Debug tensor helpers for rollout-enabled schedulers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.distributed import (
|
||||
get_local_torch_device,
|
||||
get_sp_world_size,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
|
||||
from sglang.multimodal_gen.runtime.post_training.rl_dataclasses import (
|
||||
RolloutDebugTensors,
|
||||
RolloutSessionData,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
|
||||
|
||||
|
||||
class SchedulerRLDebugMixin:
|
||||
@staticmethod
|
||||
|
||||
@@ -1,15 +1,16 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Flow-matching rollout step utilities for log-prob computation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from typing import Any, Union
|
||||
from typing import TYPE_CHECKING, Any, Union
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.distributed import (
|
||||
get_sp_world_size,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
|
||||
from sglang.multimodal_gen.runtime.post_training.rl_dataclasses import (
|
||||
RolloutSessionData,
|
||||
)
|
||||
@@ -17,6 +18,9 @@ from sglang.multimodal_gen.runtime.post_training.scheduler_rl_debug_mixin import
|
||||
SchedulerRLDebugMixin,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
|
||||
|
||||
_LOG_SQRT_2PI = math.log(math.sqrt(2 * math.pi))
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.layers.kvcache.causal_attention_cache import (
|
||||
CrossAttentionKVCache,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
|
||||
from sglang.multimodal_gen.runtime.models.dits.lingbot_world import (
|
||||
CausalLingBotWorldTransformerBlock,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.dits.longlive2 import (
|
||||
LongLive2CausalWanTransformerBlock,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.dits.wanvideo import WanT2VCrossAttention
|
||||
from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
@pytest.mark.parametrize(
|
||||
"cls", [LongLive2CausalWanTransformerBlock, CausalLingBotWorldTransformerBlock]
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
|
||||
@torch.inference_mode()
|
||||
def test_causal_cross_attention_cache(cls, dtype, single_process_model_parallel):
|
||||
block = cls.__new__(cls)
|
||||
torch.nn.Module.__init__(block)
|
||||
block.attn2 = WanT2VCrossAttention(
|
||||
64, 4, supported_attention_backends={AttentionBackendEnum.TORCH_SDPA}
|
||||
).to(device="cuda", dtype=dtype)
|
||||
for parameter in block.parameters():
|
||||
parameter.normal_(std=0.02)
|
||||
hidden = torch.randn(2, 7, 64, device="cuda", dtype=dtype)
|
||||
context = torch.randn(2, 5, 64, device="cuda", dtype=dtype)
|
||||
cache = CrossAttentionKVCache(torch.empty(0), torch.empty(0))
|
||||
with set_forward_context(current_timestep=0, attn_metadata=None):
|
||||
expected = block.attn2(hidden, context, None)
|
||||
with patch.object(
|
||||
block.attn2.to_k, "forward", wraps=block.attn2.to_k.forward
|
||||
) as project_k:
|
||||
first = block._cross_attn_with_cache(hidden, context, cache)
|
||||
keys, values = cache.k, cache.v
|
||||
repeated = block._cross_attn_with_cache(hidden, context + 1, cache)
|
||||
assert project_k.call_count == 1
|
||||
assert cache.k is keys and cache.v is values
|
||||
torch.testing.assert_close(first, expected, rtol=0, atol=0)
|
||||
torch.testing.assert_close(repeated, expected, rtol=0, atol=0)
|
||||
cache.reset()
|
||||
refreshed = block._cross_attn_with_cache(hidden, context + 1, cache)
|
||||
assert project_k.call_count == 2
|
||||
expected = block.attn2(hidden, context + 1, None)
|
||||
torch.testing.assert_close(refreshed, expected, rtol=0, atol=0)
|
||||
uncached = block._cross_attn_with_cache(hidden, context, None)
|
||||
torch.testing.assert_close(uncached, first, rtol=0, atol=0)
|
||||
@@ -0,0 +1,54 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from diffusers import EulerDiscreteScheduler
|
||||
|
||||
from sglang.multimodal_gen.runtime.models.schedulers.scheduling_flow_match_euler_discrete import (
|
||||
FlowMatchEulerDiscreteScheduler,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.schedulers.scheduling_unipc_multistep import (
|
||||
UniPCMultistepScheduler,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"cls", [FlowMatchEulerDiscreteScheduler, UniPCMultistepScheduler]
|
||||
)
|
||||
@pytest.mark.parametrize("schedule", ["karras", "exponential", "beta"])
|
||||
@pytest.mark.parametrize(
|
||||
"bounds",
|
||||
[
|
||||
{},
|
||||
{"sigma_min": None, "sigma_max": None},
|
||||
{"sigma_min": 0.02},
|
||||
{"sigma_max": 0.7},
|
||||
],
|
||||
)
|
||||
def test_sigma_schedule_conversion_and_steps(cls, schedule, bounds):
|
||||
scheduler = cls(**{f"use_{schedule}_sigmas": True})
|
||||
scheduler.register_to_config(**bounds)
|
||||
reference = EulerDiscreteScheduler()
|
||||
reference.register_to_config(**bounds)
|
||||
methods = lambda s: {
|
||||
"karras": s._convert_to_karras,
|
||||
"exponential": s._convert_to_exponential,
|
||||
"beta": s._convert_to_beta,
|
||||
}
|
||||
for steps in (1, 7):
|
||||
sigmas = torch.linspace(0.9, 0.01, 16)
|
||||
actual = methods(scheduler)[schedule](sigmas, steps)
|
||||
expected = methods(reference)[schedule](sigmas, steps)
|
||||
np.testing.assert_array_equal(actual, expected)
|
||||
outputs = []
|
||||
for _ in range(2):
|
||||
scheduler.set_timesteps(7)
|
||||
assert torch.isfinite(scheduler.sigmas).all()
|
||||
assert (scheduler.sigmas[1:] <= scheduler.sigmas[:-1]).all()
|
||||
sample = torch.linspace(-1, 1, 48).reshape(1, 3, 4, 4)
|
||||
for timestep in scheduler.timesteps:
|
||||
sample = scheduler.step(sample * 0.1, timestep, sample).prev_sample
|
||||
assert torch.isfinite(sample).all()
|
||||
outputs.append(sample)
|
||||
torch.testing.assert_close(*outputs, rtol=0, atol=0)
|
||||
Reference in New Issue
Block a user