diff --git a/python/sglang/multimodal_gen/runtime/models/dits/causal_wanvideo.py b/python/sglang/multimodal_gen/runtime/models/dits/causal_wanvideo.py index c5f0e90b55..7eb8e9b8f4 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/causal_wanvideo.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/causal_wanvideo.py @@ -330,6 +330,42 @@ class CausalWanTransformerBlock(nn.Module): self.scale_shift_table = nn.Parameter(torch.randn(1, 6, dim) / dim**0.5) + def _cross_attn_with_cache( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor, + crossattn_cache: CrossAttentionKVCache | None, + ) -> torch.Tensor: + attn2 = self.attn2 + q, _ = attn2.to_q(hidden_states) + if attn2.tp_rmsnorm: + q = tensor_parallel_rms_norm(q, attn2.norm_q) + else: + q = attn2.norm_q(q) + q = q.unflatten(2, (attn2.local_num_heads, attn2.head_dim)) + + if crossattn_cache is not None and crossattn_cache.is_init: + k = crossattn_cache.k + v = crossattn_cache.v + else: + k, _ = attn2.to_k(encoder_hidden_states) + if attn2.tp_rmsnorm: + k = tensor_parallel_rms_norm(k, attn2.norm_k) + else: + k = attn2.norm_k(k) + k = k.unflatten(2, (attn2.local_num_heads, attn2.head_dim)) + + v, _ = attn2.to_v(encoder_hidden_states) + v = v.unflatten(2, (attn2.local_num_heads, attn2.head_dim)) + + if crossattn_cache is not None: + crossattn_cache.store(k, v) + + hidden_states = attn2.attn(q, k, v) + hidden_states = hidden_states.flatten(2) + hidden_states, _ = attn2.to_out(hidden_states) + return hidden_states + def forward( self, hidden_states: torch.Tensor, diff --git a/python/sglang/multimodal_gen/runtime/models/dits/lingbot_world.py b/python/sglang/multimodal_gen/runtime/models/dits/lingbot_world.py index 70bf568550..18eca688d6 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/lingbot_world.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/lingbot_world.py @@ -1146,42 +1146,6 @@ class CausalLingBotWorldTransformerBlock(CausalWanTransformerBlock): value, _ = self.to_v(hidden_states) return query, key, value - def _cross_attn_with_cache( - self, - hidden_states: torch.Tensor, - encoder_hidden_states: torch.Tensor, - crossattn_cache: CrossAttentionKVCache | None, - ) -> torch.Tensor: - attn2 = self.attn2 - q, _ = attn2.to_q(hidden_states) - if attn2.tp_rmsnorm: - q = tensor_parallel_rms_norm(q, attn2.norm_q) - else: - q = attn2.norm_q(q) - q = q.unflatten(2, (attn2.local_num_heads, attn2.head_dim)) - - if crossattn_cache is not None and crossattn_cache.is_init: - k = crossattn_cache.k - v = crossattn_cache.v - else: - k, _ = attn2.to_k(encoder_hidden_states) - if attn2.tp_rmsnorm: - k = tensor_parallel_rms_norm(k, attn2.norm_k) - else: - k = attn2.norm_k(k) - k = k.unflatten(2, (attn2.local_num_heads, attn2.head_dim)) - - v, _ = attn2.to_v(encoder_hidden_states) - v = v.unflatten(2, (attn2.local_num_heads, attn2.head_dim)) - - if crossattn_cache is not None: - crossattn_cache.store(k, v) - - hidden_states = attn2.attn(q, k, v) - hidden_states = hidden_states.flatten(2) - hidden_states, _ = attn2.to_out(hidden_states) - return hidden_states - def _cam_conditioner_scale_shift( self, c2ws_plucker_emb: torch.Tensor | None, diff --git a/python/sglang/multimodal_gen/runtime/models/dits/longlive2.py b/python/sglang/multimodal_gen/runtime/models/dits/longlive2.py index c47acabcdf..fe05bb3a17 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/longlive2.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/longlive2.py @@ -13,7 +13,6 @@ from sglang.multimodal_gen.runtime.layers.kvcache.causal_attention_cache import ) from sglang.multimodal_gen.runtime.layers.layernorm import ( LayerNormScaleShift, - tensor_parallel_rms_norm, ) from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config import ( QuantizationConfig, @@ -34,42 +33,6 @@ class LongLive2CausalWanTransformerBlock(CausalWanTransformerBlock): dtype=torch.float32, ) - def _cross_attn_with_cache( - self, - hidden_states: torch.Tensor, - encoder_hidden_states: torch.Tensor, - crossattn_cache: CrossAttentionKVCache | None, - ) -> torch.Tensor: - attn2 = self.attn2 - q, _ = attn2.to_q(hidden_states) - if attn2.tp_rmsnorm: - q = tensor_parallel_rms_norm(q, attn2.norm_q) - else: - q = attn2.norm_q(q) - q = q.unflatten(2, (attn2.local_num_heads, attn2.head_dim)) - - if crossattn_cache is not None and crossattn_cache.is_init: - k = crossattn_cache.k - v = crossattn_cache.v - else: - k, _ = attn2.to_k(encoder_hidden_states) - if attn2.tp_rmsnorm: - k = tensor_parallel_rms_norm(k, attn2.norm_k) - else: - k = attn2.norm_k(k) - k = k.unflatten(2, (attn2.local_num_heads, attn2.head_dim)) - - v, _ = attn2.to_v(encoder_hidden_states) - v = v.unflatten(2, (attn2.local_num_heads, attn2.head_dim)) - - if crossattn_cache is not None: - crossattn_cache.store(k, v) - - hidden_states = attn2.attn(q, k, v) - hidden_states = hidden_states.flatten(2) - hidden_states, _ = attn2.to_out(hidden_states) - return hidden_states - def forward( self, hidden_states: torch.Tensor, diff --git a/python/sglang/multimodal_gen/runtime/models/schedulers/scheduling_flow_match_euler_discrete.py b/python/sglang/multimodal_gen/runtime/models/schedulers/scheduling_flow_match_euler_discrete.py index e38d4d792e..b11e1e93d7 100644 --- a/python/sglang/multimodal_gen/runtime/models/schedulers/scheduling_flow_match_euler_discrete.py +++ b/python/sglang/multimodal_gen/runtime/models/schedulers/scheduling_flow_match_euler_discrete.py @@ -25,13 +25,15 @@ from dataclasses import dataclass from typing import Any import numpy as np -import scipy.stats import torch from diffusers.configuration_utils import ConfigMixin, register_to_config from diffusers.schedulers.scheduling_utils import SchedulerMixin from diffusers.utils import BaseOutput from sglang.multimodal_gen.runtime.models.schedulers.base import BaseScheduler +from sglang.multimodal_gen.runtime.models.schedulers.sigma_schedule import ( + SigmaScheduleMixin, +) from sglang.multimodal_gen.runtime.post_training.scheduler_rl_mixin import ( SchedulerRLMixin, ) @@ -55,7 +57,7 @@ class FlowMatchEulerDiscreteSchedulerOutput(BaseOutput): class FlowMatchEulerDiscreteScheduler( - SchedulerMixin, ConfigMixin, BaseScheduler, SchedulerRLMixin + SchedulerMixin, ConfigMixin, BaseScheduler, SchedulerRLMixin, SigmaScheduleMixin ): """ Euler scheduler. @@ -369,22 +371,19 @@ class FlowMatchEulerDiscreteScheduler( # 4. If required, convert sigmas to one of karras, exponential, or beta sigma schedules if self.config.use_karras_sigmas: sigmas_tensor = torch.from_numpy(sigmas_array).to(dtype=torch.float32) - sigmas_tensor = self._convert_to_karras( + sigmas_array = self._convert_to_karras( in_sigmas=sigmas_tensor, num_inference_steps=num_inference_steps ) - sigmas_array = sigmas_tensor.numpy() elif self.config.use_exponential_sigmas: sigmas_tensor = torch.from_numpy(sigmas_array).to(dtype=torch.float32) - sigmas_tensor = self._convert_to_exponential( + sigmas_array = self._convert_to_exponential( in_sigmas=sigmas_tensor, num_inference_steps=num_inference_steps ) - sigmas_array = sigmas_tensor.numpy() elif self.config.use_beta_sigmas: sigmas_tensor = torch.from_numpy(sigmas_array).to(dtype=torch.float32) - sigmas_tensor = self._convert_to_beta( + sigmas_array = self._convert_to_beta( in_sigmas=sigmas_tensor, num_inference_steps=num_inference_steps ) - sigmas_array = sigmas_tensor.numpy() # 5. Convert sigmas and timesteps to tensors and move to specified device sigmas_tensor = torch.from_numpy(sigmas_array).to( @@ -555,96 +554,6 @@ class FlowMatchEulerDiscreteScheduler( return FlowMatchEulerDiscreteSchedulerOutput(prev_sample=prev_sample) - # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_karras - def _convert_to_karras( - self, in_sigmas: torch.Tensor, num_inference_steps: int - ) -> torch.Tensor: - """Constructs the noise schedule of Karras et al. (2022).""" - - # Hack to make sure that other schedulers which copy this function don't break - # TODO: Add this logic to the other schedulers - if hasattr(self.config, "sigma_min"): - sigma_min = self.config.sigma_min - else: - sigma_min = None - - if hasattr(self.config, "sigma_max"): - sigma_max = self.config.sigma_max - else: - sigma_max = None - - sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item() - sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item() - - rho = 7.0 # 7.0 is the value used in the paper - ramp = np.linspace(0, 1, num_inference_steps) - min_inv_rho = sigma_min ** (1 / rho) - max_inv_rho = sigma_max ** (1 / rho) - sigmas = (max_inv_rho + ramp * (min_inv_rho - max_inv_rho)) ** rho - return sigmas - - # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_exponential - def _convert_to_exponential( - self, in_sigmas: torch.Tensor, num_inference_steps: int - ) -> torch.Tensor: - """Constructs an exponential noise schedule.""" - - # Hack to make sure that other schedulers which copy this function don't break - # TODO: Add this logic to the other schedulers - if hasattr(self.config, "sigma_min"): - sigma_min = self.config.sigma_min - else: - sigma_min = None - - if hasattr(self.config, "sigma_max"): - sigma_max = self.config.sigma_max - else: - sigma_max = None - - sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item() - sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item() - - sigmas = np.exp( - np.linspace(math.log(sigma_max), math.log(sigma_min), num_inference_steps) - ) - return sigmas - - # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_beta - def _convert_to_beta( - self, - in_sigmas: torch.Tensor, - num_inference_steps: int, - alpha: float = 0.6, - beta: float = 0.6, - ) -> torch.Tensor: - """From "Beta Sampling is All You Need" [arXiv:2407.12173] (Lee et. al, 2024)""" - - # Hack to make sure that other schedulers which copy this function don't break - # TODO: Add this logic to the other schedulers - if hasattr(self.config, "sigma_min"): - sigma_min = self.config.sigma_min - else: - sigma_min = None - - if hasattr(self.config, "sigma_max"): - sigma_max = self.config.sigma_max - else: - sigma_max = None - - sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item() - sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item() - - sigmas = np.array( - [ - sigma_min + (ppf * (sigma_max - sigma_min)) - for ppf in [ - scipy.stats.beta.ppf(timestep, alpha, beta) - for timestep in 1 - np.linspace(0, 1, num_inference_steps) - ] - ] - ) - return sigmas - def _time_shift_exponential( self, mu: float, sigma: float, t: torch.Tensor | np.ndarray ) -> torch.Tensor | np.ndarray: diff --git a/python/sglang/multimodal_gen/runtime/models/schedulers/scheduling_unipc_multistep.py b/python/sglang/multimodal_gen/runtime/models/schedulers/scheduling_unipc_multistep.py index fd7e0b1d90..de84315e4b 100644 --- a/python/sglang/multimodal_gen/runtime/models/schedulers/scheduling_unipc_multistep.py +++ b/python/sglang/multimodal_gen/runtime/models/schedulers/scheduling_unipc_multistep.py @@ -35,9 +35,9 @@ from diffusers.schedulers.scheduling_utils import ( from diffusers.utils import deprecate, is_scipy_available from sglang.multimodal_gen.runtime.models.schedulers.base import BaseScheduler - -if is_scipy_available(): - import scipy.stats +from sglang.multimodal_gen.runtime.models.schedulers.sigma_schedule import ( + SigmaScheduleMixin, +) # Copied from diffusers.schedulers.scheduling_ddpm.betas_for_alpha_bar @@ -122,7 +122,9 @@ def rescale_zero_terminal_snr(betas): return betas -class UniPCMultistepScheduler(SchedulerMixin, ConfigMixin, BaseScheduler): +class UniPCMultistepScheduler( + SchedulerMixin, ConfigMixin, BaseScheduler, SigmaScheduleMixin +): """ `UniPCMultistepScheduler` is a training-free framework designed for the fast sampling of diffusion models. @@ -614,96 +616,6 @@ class UniPCMultistepScheduler(SchedulerMixin, ConfigMixin, BaseScheduler): return alpha_t, sigma_t - # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_karras - def _convert_to_karras( - self, in_sigmas: torch.Tensor, num_inference_steps - ) -> torch.Tensor: - """Constructs the noise schedule of Karras et al. (2022).""" - - # Hack to make sure that other schedulers which copy this function don't break - # TODO: Add this logic to the other schedulers - if hasattr(self.config, "sigma_min"): - sigma_min = self.config.sigma_min - else: - sigma_min = None - - if hasattr(self.config, "sigma_max"): - sigma_max = self.config.sigma_max - else: - sigma_max = None - - sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item() - sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item() - - rho = 7.0 # 7.0 is the value used in the paper - ramp = np.linspace(0, 1, num_inference_steps) - min_inv_rho = sigma_min ** (1 / rho) - max_inv_rho = sigma_max ** (1 / rho) - sigmas = (max_inv_rho + ramp * (min_inv_rho - max_inv_rho)) ** rho - return sigmas - - # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_exponential - def _convert_to_exponential( - self, in_sigmas: torch.Tensor, num_inference_steps: int - ) -> torch.Tensor: - """Constructs an exponential noise schedule.""" - - # Hack to make sure that other schedulers which copy this function don't break - # TODO: Add this logic to the other schedulers - if hasattr(self.config, "sigma_min"): - sigma_min = self.config.sigma_min - else: - sigma_min = None - - if hasattr(self.config, "sigma_max"): - sigma_max = self.config.sigma_max - else: - sigma_max = None - - sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item() - sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item() - - sigmas = np.exp( - np.linspace(math.log(sigma_max), math.log(sigma_min), num_inference_steps) - ) - return sigmas - - # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_beta - def _convert_to_beta( - self, - in_sigmas: torch.Tensor, - num_inference_steps: int, - alpha: float = 0.6, - beta: float = 0.6, - ) -> torch.Tensor: - """From "Beta Sampling is All You Need" [arXiv:2407.12173] (Lee et. al, 2024)""" - - # Hack to make sure that other schedulers which copy this function don't break - # TODO: Add this logic to the other schedulers - if hasattr(self.config, "sigma_min"): - sigma_min = self.config.sigma_min - else: - sigma_min = None - - if hasattr(self.config, "sigma_max"): - sigma_max = self.config.sigma_max - else: - sigma_max = None - - sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item() - sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item() - - sigmas = np.array( - [ - sigma_min + (ppf * (sigma_max - sigma_min)) - for ppf in [ - scipy.stats.beta.ppf(timestep, alpha, beta) - for timestep in 1 - np.linspace(0, 1, num_inference_steps) - ] - ] - ) - return sigmas - def convert_model_output( self, model_output: torch.Tensor, diff --git a/python/sglang/multimodal_gen/runtime/models/schedulers/sigma_schedule.py b/python/sglang/multimodal_gen/runtime/models/schedulers/sigma_schedule.py new file mode 100644 index 0000000000..c5256b08a2 --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/models/schedulers/sigma_schedule.py @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2024 Stability AI, Katherine Crowson and The HuggingFace Team. +# Copyright 2025 TSAIL Team and The HuggingFace Team. +# Adapted from diffusers.schedulers.scheduling_euler_discrete. + +import math + +import numpy as np +import torch +from diffusers.utils import is_scipy_available + +if is_scipy_available(): + import scipy.stats + + +class SigmaScheduleMixin: + """Alternative sigma schedules for schedulers with a Diffusers config.""" + + def _sigma_bounds(self, in_sigmas): + sigma_min = self.config.get("sigma_min") + sigma_max = self.config.get("sigma_max") + sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item() + sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item() + return sigma_min, sigma_max + + def _convert_to_karras( + self, in_sigmas: torch.Tensor, num_inference_steps: int + ) -> np.ndarray: + """Construct the Karras et al. (2022) noise schedule.""" + sigma_min, sigma_max = self._sigma_bounds(in_sigmas) + rho = 7.0 + ramp = np.linspace(0, 1, num_inference_steps) + min_inv_rho = sigma_min ** (1 / rho) + max_inv_rho = sigma_max ** (1 / rho) + return (max_inv_rho + ramp * (min_inv_rho - max_inv_rho)) ** rho + + def _convert_to_exponential( + self, in_sigmas: torch.Tensor, num_inference_steps: int + ) -> np.ndarray: + """Construct an exponential noise schedule.""" + sigma_min, sigma_max = self._sigma_bounds(in_sigmas) + return np.exp( + np.linspace(math.log(sigma_max), math.log(sigma_min), num_inference_steps) + ) + + def _convert_to_beta( + self, + in_sigmas: torch.Tensor, + num_inference_steps: int, + alpha: float = 0.6, + beta: float = 0.6, + ) -> np.ndarray: + """Construct the Beta Sampling (Lee et al., 2024) noise schedule.""" + sigma_min, sigma_max = self._sigma_bounds(in_sigmas) + return np.array( + [ + sigma_min + (ppf * (sigma_max - sigma_min)) + for ppf in [ + scipy.stats.beta.ppf(timestep, alpha, beta) + for timestep in 1 - np.linspace(0, 1, num_inference_steps) + ] + ] + ) diff --git a/python/sglang/multimodal_gen/runtime/post_training/scheduler_rl_debug_mixin.py b/python/sglang/multimodal_gen/runtime/post_training/scheduler_rl_debug_mixin.py index e6113876ae..b6e1222a45 100644 --- a/python/sglang/multimodal_gen/runtime/post_training/scheduler_rl_debug_mixin.py +++ b/python/sglang/multimodal_gen/runtime/post_training/scheduler_rl_debug_mixin.py @@ -1,18 +1,24 @@ # SPDX-License-Identifier: Apache-2.0 """Debug tensor helpers for rollout-enabled schedulers.""" +from __future__ import annotations + +from typing import TYPE_CHECKING + import torch from sglang.multimodal_gen.runtime.distributed import ( get_local_torch_device, get_sp_world_size, ) -from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req from sglang.multimodal_gen.runtime.post_training.rl_dataclasses import ( RolloutDebugTensors, RolloutSessionData, ) +if TYPE_CHECKING: + from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req + class SchedulerRLDebugMixin: @staticmethod diff --git a/python/sglang/multimodal_gen/runtime/post_training/scheduler_rl_mixin.py b/python/sglang/multimodal_gen/runtime/post_training/scheduler_rl_mixin.py index aa52a25e65..af1c520f10 100644 --- a/python/sglang/multimodal_gen/runtime/post_training/scheduler_rl_mixin.py +++ b/python/sglang/multimodal_gen/runtime/post_training/scheduler_rl_mixin.py @@ -1,15 +1,16 @@ # SPDX-License-Identifier: Apache-2.0 """Flow-matching rollout step utilities for log-prob computation.""" +from __future__ import annotations + import math -from typing import Any, Union +from typing import TYPE_CHECKING, Any, Union import torch from sglang.multimodal_gen.runtime.distributed import ( get_sp_world_size, ) -from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req from sglang.multimodal_gen.runtime.post_training.rl_dataclasses import ( RolloutSessionData, ) @@ -17,6 +18,9 @@ from sglang.multimodal_gen.runtime.post_training.scheduler_rl_debug_mixin import SchedulerRLDebugMixin, ) +if TYPE_CHECKING: + from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req + _LOG_SQRT_2PI = math.log(math.sqrt(2 * math.pi)) diff --git a/python/sglang/multimodal_gen/test/unit/realtime/test_causal_cross_attention.py b/python/sglang/multimodal_gen/test/unit/realtime/test_causal_cross_attention.py new file mode 100644 index 0000000000..5e183d469f --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/realtime/test_causal_cross_attention.py @@ -0,0 +1,57 @@ +# SPDX-License-Identifier: Apache-2.0 + +from unittest.mock import patch + +import pytest +import torch + +from sglang.multimodal_gen.runtime.layers.kvcache.causal_attention_cache import ( + CrossAttentionKVCache, +) +from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context +from sglang.multimodal_gen.runtime.models.dits.lingbot_world import ( + CausalLingBotWorldTransformerBlock, +) +from sglang.multimodal_gen.runtime.models.dits.longlive2 import ( + LongLive2CausalWanTransformerBlock, +) +from sglang.multimodal_gen.runtime.models.dits.wanvideo import WanT2VCrossAttention +from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +@pytest.mark.parametrize( + "cls", [LongLive2CausalWanTransformerBlock, CausalLingBotWorldTransformerBlock] +) +@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16]) +@torch.inference_mode() +def test_causal_cross_attention_cache(cls, dtype, single_process_model_parallel): + block = cls.__new__(cls) + torch.nn.Module.__init__(block) + block.attn2 = WanT2VCrossAttention( + 64, 4, supported_attention_backends={AttentionBackendEnum.TORCH_SDPA} + ).to(device="cuda", dtype=dtype) + for parameter in block.parameters(): + parameter.normal_(std=0.02) + hidden = torch.randn(2, 7, 64, device="cuda", dtype=dtype) + context = torch.randn(2, 5, 64, device="cuda", dtype=dtype) + cache = CrossAttentionKVCache(torch.empty(0), torch.empty(0)) + with set_forward_context(current_timestep=0, attn_metadata=None): + expected = block.attn2(hidden, context, None) + with patch.object( + block.attn2.to_k, "forward", wraps=block.attn2.to_k.forward + ) as project_k: + first = block._cross_attn_with_cache(hidden, context, cache) + keys, values = cache.k, cache.v + repeated = block._cross_attn_with_cache(hidden, context + 1, cache) + assert project_k.call_count == 1 + assert cache.k is keys and cache.v is values + torch.testing.assert_close(first, expected, rtol=0, atol=0) + torch.testing.assert_close(repeated, expected, rtol=0, atol=0) + cache.reset() + refreshed = block._cross_attn_with_cache(hidden, context + 1, cache) + assert project_k.call_count == 2 + expected = block.attn2(hidden, context + 1, None) + torch.testing.assert_close(refreshed, expected, rtol=0, atol=0) + uncached = block._cross_attn_with_cache(hidden, context, None) + torch.testing.assert_close(uncached, first, rtol=0, atol=0) diff --git a/python/sglang/multimodal_gen/test/unit/test_sigma_schedule.py b/python/sglang/multimodal_gen/test/unit/test_sigma_schedule.py new file mode 100644 index 0000000000..a83ba80350 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_sigma_schedule.py @@ -0,0 +1,54 @@ +# SPDX-License-Identifier: Apache-2.0 + +import numpy as np +import pytest +import torch +from diffusers import EulerDiscreteScheduler + +from sglang.multimodal_gen.runtime.models.schedulers.scheduling_flow_match_euler_discrete import ( + FlowMatchEulerDiscreteScheduler, +) +from sglang.multimodal_gen.runtime.models.schedulers.scheduling_unipc_multistep import ( + UniPCMultistepScheduler, +) + + +@pytest.mark.parametrize( + "cls", [FlowMatchEulerDiscreteScheduler, UniPCMultistepScheduler] +) +@pytest.mark.parametrize("schedule", ["karras", "exponential", "beta"]) +@pytest.mark.parametrize( + "bounds", + [ + {}, + {"sigma_min": None, "sigma_max": None}, + {"sigma_min": 0.02}, + {"sigma_max": 0.7}, + ], +) +def test_sigma_schedule_conversion_and_steps(cls, schedule, bounds): + scheduler = cls(**{f"use_{schedule}_sigmas": True}) + scheduler.register_to_config(**bounds) + reference = EulerDiscreteScheduler() + reference.register_to_config(**bounds) + methods = lambda s: { + "karras": s._convert_to_karras, + "exponential": s._convert_to_exponential, + "beta": s._convert_to_beta, + } + for steps in (1, 7): + sigmas = torch.linspace(0.9, 0.01, 16) + actual = methods(scheduler)[schedule](sigmas, steps) + expected = methods(reference)[schedule](sigmas, steps) + np.testing.assert_array_equal(actual, expected) + outputs = [] + for _ in range(2): + scheduler.set_timesteps(7) + assert torch.isfinite(scheduler.sigmas).all() + assert (scheduler.sigmas[1:] <= scheduler.sigmas[:-1]).all() + sample = torch.linspace(-1, 1, 48).reshape(1, 3, 4, 4) + for timestep in scheduler.timesteps: + sample = scheduler.step(sample * 0.1, timestep, sample).prev_sample + assert torch.isfinite(sample).all() + outputs.append(sample) + torch.testing.assert_close(*outputs, rtol=0, atol=0)