[diffusion] refactor: share sigma schedules and causal cross-attention (#42848)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
Mick
2026-10-08 09:29:33 +08:00
committed by GitHub
co-authored by Mick Qian
parent 28c08b02c3
commit 21fa2251a0
10 changed files with 236 additions and 268 deletions
@@ -330,6 +330,42 @@ class CausalWanTransformerBlock(nn.Module):
self.scale_shift_table = nn.Parameter(torch.randn(1, 6, dim) / dim**0.5)
def _cross_attn_with_cache(
self,
hidden_states: torch.Tensor,
encoder_hidden_states: torch.Tensor,
crossattn_cache: CrossAttentionKVCache | None,
) -> torch.Tensor:
attn2 = self.attn2
q, _ = attn2.to_q(hidden_states)
if attn2.tp_rmsnorm:
q = tensor_parallel_rms_norm(q, attn2.norm_q)
else:
q = attn2.norm_q(q)
q = q.unflatten(2, (attn2.local_num_heads, attn2.head_dim))
if crossattn_cache is not None and crossattn_cache.is_init:
k = crossattn_cache.k
v = crossattn_cache.v
else:
k, _ = attn2.to_k(encoder_hidden_states)
if attn2.tp_rmsnorm:
k = tensor_parallel_rms_norm(k, attn2.norm_k)
else:
k = attn2.norm_k(k)
k = k.unflatten(2, (attn2.local_num_heads, attn2.head_dim))
v, _ = attn2.to_v(encoder_hidden_states)
v = v.unflatten(2, (attn2.local_num_heads, attn2.head_dim))
if crossattn_cache is not None:
crossattn_cache.store(k, v)
hidden_states = attn2.attn(q, k, v)
hidden_states = hidden_states.flatten(2)
hidden_states, _ = attn2.to_out(hidden_states)
return hidden_states
def forward(
self,
hidden_states: torch.Tensor,
@@ -1146,42 +1146,6 @@ class CausalLingBotWorldTransformerBlock(CausalWanTransformerBlock):
value, _ = self.to_v(hidden_states)
return query, key, value
def _cross_attn_with_cache(
self,
hidden_states: torch.Tensor,
encoder_hidden_states: torch.Tensor,
crossattn_cache: CrossAttentionKVCache | None,
) -> torch.Tensor:
attn2 = self.attn2
q, _ = attn2.to_q(hidden_states)
if attn2.tp_rmsnorm:
q = tensor_parallel_rms_norm(q, attn2.norm_q)
else:
q = attn2.norm_q(q)
q = q.unflatten(2, (attn2.local_num_heads, attn2.head_dim))
if crossattn_cache is not None and crossattn_cache.is_init:
k = crossattn_cache.k
v = crossattn_cache.v
else:
k, _ = attn2.to_k(encoder_hidden_states)
if attn2.tp_rmsnorm:
k = tensor_parallel_rms_norm(k, attn2.norm_k)
else:
k = attn2.norm_k(k)
k = k.unflatten(2, (attn2.local_num_heads, attn2.head_dim))
v, _ = attn2.to_v(encoder_hidden_states)
v = v.unflatten(2, (attn2.local_num_heads, attn2.head_dim))
if crossattn_cache is not None:
crossattn_cache.store(k, v)
hidden_states = attn2.attn(q, k, v)
hidden_states = hidden_states.flatten(2)
hidden_states, _ = attn2.to_out(hidden_states)
return hidden_states
def _cam_conditioner_scale_shift(
self,
c2ws_plucker_emb: torch.Tensor | None,
@@ -13,7 +13,6 @@ from sglang.multimodal_gen.runtime.layers.kvcache.causal_attention_cache import
)
from sglang.multimodal_gen.runtime.layers.layernorm import (
LayerNormScaleShift,
tensor_parallel_rms_norm,
)
from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config import (
QuantizationConfig,
@@ -34,42 +33,6 @@ class LongLive2CausalWanTransformerBlock(CausalWanTransformerBlock):
dtype=torch.float32,
)
def _cross_attn_with_cache(
self,
hidden_states: torch.Tensor,
encoder_hidden_states: torch.Tensor,
crossattn_cache: CrossAttentionKVCache | None,
) -> torch.Tensor:
attn2 = self.attn2
q, _ = attn2.to_q(hidden_states)
if attn2.tp_rmsnorm:
q = tensor_parallel_rms_norm(q, attn2.norm_q)
else:
q = attn2.norm_q(q)
q = q.unflatten(2, (attn2.local_num_heads, attn2.head_dim))
if crossattn_cache is not None and crossattn_cache.is_init:
k = crossattn_cache.k
v = crossattn_cache.v
else:
k, _ = attn2.to_k(encoder_hidden_states)
if attn2.tp_rmsnorm:
k = tensor_parallel_rms_norm(k, attn2.norm_k)
else:
k = attn2.norm_k(k)
k = k.unflatten(2, (attn2.local_num_heads, attn2.head_dim))
v, _ = attn2.to_v(encoder_hidden_states)
v = v.unflatten(2, (attn2.local_num_heads, attn2.head_dim))
if crossattn_cache is not None:
crossattn_cache.store(k, v)
hidden_states = attn2.attn(q, k, v)
hidden_states = hidden_states.flatten(2)
hidden_states, _ = attn2.to_out(hidden_states)
return hidden_states
def forward(
self,
hidden_states: torch.Tensor,
@@ -25,13 +25,15 @@ from dataclasses import dataclass
from typing import Any
import numpy as np
import scipy.stats
import torch
from diffusers.configuration_utils import ConfigMixin, register_to_config
from diffusers.schedulers.scheduling_utils import SchedulerMixin
from diffusers.utils import BaseOutput
from sglang.multimodal_gen.runtime.models.schedulers.base import BaseScheduler
from sglang.multimodal_gen.runtime.models.schedulers.sigma_schedule import (
SigmaScheduleMixin,
)
from sglang.multimodal_gen.runtime.post_training.scheduler_rl_mixin import (
SchedulerRLMixin,
)
@@ -55,7 +57,7 @@ class FlowMatchEulerDiscreteSchedulerOutput(BaseOutput):
class FlowMatchEulerDiscreteScheduler(
SchedulerMixin, ConfigMixin, BaseScheduler, SchedulerRLMixin
SchedulerMixin, ConfigMixin, BaseScheduler, SchedulerRLMixin, SigmaScheduleMixin
):
"""
Euler scheduler.
@@ -369,22 +371,19 @@ class FlowMatchEulerDiscreteScheduler(
# 4. If required, convert sigmas to one of karras, exponential, or beta sigma schedules
if self.config.use_karras_sigmas:
sigmas_tensor = torch.from_numpy(sigmas_array).to(dtype=torch.float32)
sigmas_tensor = self._convert_to_karras(
sigmas_array = self._convert_to_karras(
in_sigmas=sigmas_tensor, num_inference_steps=num_inference_steps
)
sigmas_array = sigmas_tensor.numpy()
elif self.config.use_exponential_sigmas:
sigmas_tensor = torch.from_numpy(sigmas_array).to(dtype=torch.float32)
sigmas_tensor = self._convert_to_exponential(
sigmas_array = self._convert_to_exponential(
in_sigmas=sigmas_tensor, num_inference_steps=num_inference_steps
)
sigmas_array = sigmas_tensor.numpy()
elif self.config.use_beta_sigmas:
sigmas_tensor = torch.from_numpy(sigmas_array).to(dtype=torch.float32)
sigmas_tensor = self._convert_to_beta(
sigmas_array = self._convert_to_beta(
in_sigmas=sigmas_tensor, num_inference_steps=num_inference_steps
)
sigmas_array = sigmas_tensor.numpy()
# 5. Convert sigmas and timesteps to tensors and move to specified device
sigmas_tensor = torch.from_numpy(sigmas_array).to(
@@ -555,96 +554,6 @@ class FlowMatchEulerDiscreteScheduler(
return FlowMatchEulerDiscreteSchedulerOutput(prev_sample=prev_sample)
# Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_karras
def _convert_to_karras(
self, in_sigmas: torch.Tensor, num_inference_steps: int
) -> torch.Tensor:
"""Constructs the noise schedule of Karras et al. (2022)."""
# Hack to make sure that other schedulers which copy this function don't break
# TODO: Add this logic to the other schedulers
if hasattr(self.config, "sigma_min"):
sigma_min = self.config.sigma_min
else:
sigma_min = None
if hasattr(self.config, "sigma_max"):
sigma_max = self.config.sigma_max
else:
sigma_max = None
sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item()
sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item()
rho = 7.0 # 7.0 is the value used in the paper
ramp = np.linspace(0, 1, num_inference_steps)
min_inv_rho = sigma_min ** (1 / rho)
max_inv_rho = sigma_max ** (1 / rho)
sigmas = (max_inv_rho + ramp * (min_inv_rho - max_inv_rho)) ** rho
return sigmas
# Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_exponential
def _convert_to_exponential(
self, in_sigmas: torch.Tensor, num_inference_steps: int
) -> torch.Tensor:
"""Constructs an exponential noise schedule."""
# Hack to make sure that other schedulers which copy this function don't break
# TODO: Add this logic to the other schedulers
if hasattr(self.config, "sigma_min"):
sigma_min = self.config.sigma_min
else:
sigma_min = None
if hasattr(self.config, "sigma_max"):
sigma_max = self.config.sigma_max
else:
sigma_max = None
sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item()
sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item()
sigmas = np.exp(
np.linspace(math.log(sigma_max), math.log(sigma_min), num_inference_steps)
)
return sigmas
# Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_beta
def _convert_to_beta(
self,
in_sigmas: torch.Tensor,
num_inference_steps: int,
alpha: float = 0.6,
beta: float = 0.6,
) -> torch.Tensor:
"""From "Beta Sampling is All You Need" [arXiv:2407.12173] (Lee et. al, 2024)"""
# Hack to make sure that other schedulers which copy this function don't break
# TODO: Add this logic to the other schedulers
if hasattr(self.config, "sigma_min"):
sigma_min = self.config.sigma_min
else:
sigma_min = None
if hasattr(self.config, "sigma_max"):
sigma_max = self.config.sigma_max
else:
sigma_max = None
sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item()
sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item()
sigmas = np.array(
[
sigma_min + (ppf * (sigma_max - sigma_min))
for ppf in [
scipy.stats.beta.ppf(timestep, alpha, beta)
for timestep in 1 - np.linspace(0, 1, num_inference_steps)
]
]
)
return sigmas
def _time_shift_exponential(
self, mu: float, sigma: float, t: torch.Tensor | np.ndarray
) -> torch.Tensor | np.ndarray:
@@ -35,9 +35,9 @@ from diffusers.schedulers.scheduling_utils import (
from diffusers.utils import deprecate, is_scipy_available
from sglang.multimodal_gen.runtime.models.schedulers.base import BaseScheduler
if is_scipy_available():
import scipy.stats
from sglang.multimodal_gen.runtime.models.schedulers.sigma_schedule import (
SigmaScheduleMixin,
)
# Copied from diffusers.schedulers.scheduling_ddpm.betas_for_alpha_bar
@@ -122,7 +122,9 @@ def rescale_zero_terminal_snr(betas):
return betas
class UniPCMultistepScheduler(SchedulerMixin, ConfigMixin, BaseScheduler):
class UniPCMultistepScheduler(
SchedulerMixin, ConfigMixin, BaseScheduler, SigmaScheduleMixin
):
"""
`UniPCMultistepScheduler` is a training-free framework designed for the fast sampling of diffusion models.
@@ -614,96 +616,6 @@ class UniPCMultistepScheduler(SchedulerMixin, ConfigMixin, BaseScheduler):
return alpha_t, sigma_t
# Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_karras
def _convert_to_karras(
self, in_sigmas: torch.Tensor, num_inference_steps
) -> torch.Tensor:
"""Constructs the noise schedule of Karras et al. (2022)."""
# Hack to make sure that other schedulers which copy this function don't break
# TODO: Add this logic to the other schedulers
if hasattr(self.config, "sigma_min"):
sigma_min = self.config.sigma_min
else:
sigma_min = None
if hasattr(self.config, "sigma_max"):
sigma_max = self.config.sigma_max
else:
sigma_max = None
sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item()
sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item()
rho = 7.0 # 7.0 is the value used in the paper
ramp = np.linspace(0, 1, num_inference_steps)
min_inv_rho = sigma_min ** (1 / rho)
max_inv_rho = sigma_max ** (1 / rho)
sigmas = (max_inv_rho + ramp * (min_inv_rho - max_inv_rho)) ** rho
return sigmas
# Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_exponential
def _convert_to_exponential(
self, in_sigmas: torch.Tensor, num_inference_steps: int
) -> torch.Tensor:
"""Constructs an exponential noise schedule."""
# Hack to make sure that other schedulers which copy this function don't break
# TODO: Add this logic to the other schedulers
if hasattr(self.config, "sigma_min"):
sigma_min = self.config.sigma_min
else:
sigma_min = None
if hasattr(self.config, "sigma_max"):
sigma_max = self.config.sigma_max
else:
sigma_max = None
sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item()
sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item()
sigmas = np.exp(
np.linspace(math.log(sigma_max), math.log(sigma_min), num_inference_steps)
)
return sigmas
# Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_beta
def _convert_to_beta(
self,
in_sigmas: torch.Tensor,
num_inference_steps: int,
alpha: float = 0.6,
beta: float = 0.6,
) -> torch.Tensor:
"""From "Beta Sampling is All You Need" [arXiv:2407.12173] (Lee et. al, 2024)"""
# Hack to make sure that other schedulers which copy this function don't break
# TODO: Add this logic to the other schedulers
if hasattr(self.config, "sigma_min"):
sigma_min = self.config.sigma_min
else:
sigma_min = None
if hasattr(self.config, "sigma_max"):
sigma_max = self.config.sigma_max
else:
sigma_max = None
sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item()
sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item()
sigmas = np.array(
[
sigma_min + (ppf * (sigma_max - sigma_min))
for ppf in [
scipy.stats.beta.ppf(timestep, alpha, beta)
for timestep in 1 - np.linspace(0, 1, num_inference_steps)
]
]
)
return sigmas
def convert_model_output(
self,
model_output: torch.Tensor,
@@ -0,0 +1,63 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright 2024 Stability AI, Katherine Crowson and The HuggingFace Team.
# Copyright 2025 TSAIL Team and The HuggingFace Team.
# Adapted from diffusers.schedulers.scheduling_euler_discrete.
import math
import numpy as np
import torch
from diffusers.utils import is_scipy_available
if is_scipy_available():
import scipy.stats
class SigmaScheduleMixin:
"""Alternative sigma schedules for schedulers with a Diffusers config."""
def _sigma_bounds(self, in_sigmas):
sigma_min = self.config.get("sigma_min")
sigma_max = self.config.get("sigma_max")
sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item()
sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item()
return sigma_min, sigma_max
def _convert_to_karras(
self, in_sigmas: torch.Tensor, num_inference_steps: int
) -> np.ndarray:
"""Construct the Karras et al. (2022) noise schedule."""
sigma_min, sigma_max = self._sigma_bounds(in_sigmas)
rho = 7.0
ramp = np.linspace(0, 1, num_inference_steps)
min_inv_rho = sigma_min ** (1 / rho)
max_inv_rho = sigma_max ** (1 / rho)
return (max_inv_rho + ramp * (min_inv_rho - max_inv_rho)) ** rho
def _convert_to_exponential(
self, in_sigmas: torch.Tensor, num_inference_steps: int
) -> np.ndarray:
"""Construct an exponential noise schedule."""
sigma_min, sigma_max = self._sigma_bounds(in_sigmas)
return np.exp(
np.linspace(math.log(sigma_max), math.log(sigma_min), num_inference_steps)
)
def _convert_to_beta(
self,
in_sigmas: torch.Tensor,
num_inference_steps: int,
alpha: float = 0.6,
beta: float = 0.6,
) -> np.ndarray:
"""Construct the Beta Sampling (Lee et al., 2024) noise schedule."""
sigma_min, sigma_max = self._sigma_bounds(in_sigmas)
return np.array(
[
sigma_min + (ppf * (sigma_max - sigma_min))
for ppf in [
scipy.stats.beta.ppf(timestep, alpha, beta)
for timestep in 1 - np.linspace(0, 1, num_inference_steps)
]
]
)
@@ -1,18 +1,24 @@
# SPDX-License-Identifier: Apache-2.0
"""Debug tensor helpers for rollout-enabled schedulers."""
from __future__ import annotations
from typing import TYPE_CHECKING
import torch
from sglang.multimodal_gen.runtime.distributed import (
get_local_torch_device,
get_sp_world_size,
)
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
from sglang.multimodal_gen.runtime.post_training.rl_dataclasses import (
RolloutDebugTensors,
RolloutSessionData,
)
if TYPE_CHECKING:
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
class SchedulerRLDebugMixin:
@staticmethod
@@ -1,15 +1,16 @@
# SPDX-License-Identifier: Apache-2.0
"""Flow-matching rollout step utilities for log-prob computation."""
from __future__ import annotations
import math
from typing import Any, Union
from typing import TYPE_CHECKING, Any, Union
import torch
from sglang.multimodal_gen.runtime.distributed import (
get_sp_world_size,
)
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
from sglang.multimodal_gen.runtime.post_training.rl_dataclasses import (
RolloutSessionData,
)
@@ -17,6 +18,9 @@ from sglang.multimodal_gen.runtime.post_training.scheduler_rl_debug_mixin import
SchedulerRLDebugMixin,
)
if TYPE_CHECKING:
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
_LOG_SQRT_2PI = math.log(math.sqrt(2 * math.pi))
@@ -0,0 +1,57 @@
# SPDX-License-Identifier: Apache-2.0
from unittest.mock import patch
import pytest
import torch
from sglang.multimodal_gen.runtime.layers.kvcache.causal_attention_cache import (
CrossAttentionKVCache,
)
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
from sglang.multimodal_gen.runtime.models.dits.lingbot_world import (
CausalLingBotWorldTransformerBlock,
)
from sglang.multimodal_gen.runtime.models.dits.longlive2 import (
LongLive2CausalWanTransformerBlock,
)
from sglang.multimodal_gen.runtime.models.dits.wanvideo import WanT2VCrossAttention
from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
@pytest.mark.parametrize(
"cls", [LongLive2CausalWanTransformerBlock, CausalLingBotWorldTransformerBlock]
)
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
@torch.inference_mode()
def test_causal_cross_attention_cache(cls, dtype, single_process_model_parallel):
block = cls.__new__(cls)
torch.nn.Module.__init__(block)
block.attn2 = WanT2VCrossAttention(
64, 4, supported_attention_backends={AttentionBackendEnum.TORCH_SDPA}
).to(device="cuda", dtype=dtype)
for parameter in block.parameters():
parameter.normal_(std=0.02)
hidden = torch.randn(2, 7, 64, device="cuda", dtype=dtype)
context = torch.randn(2, 5, 64, device="cuda", dtype=dtype)
cache = CrossAttentionKVCache(torch.empty(0), torch.empty(0))
with set_forward_context(current_timestep=0, attn_metadata=None):
expected = block.attn2(hidden, context, None)
with patch.object(
block.attn2.to_k, "forward", wraps=block.attn2.to_k.forward
) as project_k:
first = block._cross_attn_with_cache(hidden, context, cache)
keys, values = cache.k, cache.v
repeated = block._cross_attn_with_cache(hidden, context + 1, cache)
assert project_k.call_count == 1
assert cache.k is keys and cache.v is values
torch.testing.assert_close(first, expected, rtol=0, atol=0)
torch.testing.assert_close(repeated, expected, rtol=0, atol=0)
cache.reset()
refreshed = block._cross_attn_with_cache(hidden, context + 1, cache)
assert project_k.call_count == 2
expected = block.attn2(hidden, context + 1, None)
torch.testing.assert_close(refreshed, expected, rtol=0, atol=0)
uncached = block._cross_attn_with_cache(hidden, context, None)
torch.testing.assert_close(uncached, first, rtol=0, atol=0)
@@ -0,0 +1,54 @@
# SPDX-License-Identifier: Apache-2.0
import numpy as np
import pytest
import torch
from diffusers import EulerDiscreteScheduler
from sglang.multimodal_gen.runtime.models.schedulers.scheduling_flow_match_euler_discrete import (
FlowMatchEulerDiscreteScheduler,
)
from sglang.multimodal_gen.runtime.models.schedulers.scheduling_unipc_multistep import (
UniPCMultistepScheduler,
)
@pytest.mark.parametrize(
"cls", [FlowMatchEulerDiscreteScheduler, UniPCMultistepScheduler]
)
@pytest.mark.parametrize("schedule", ["karras", "exponential", "beta"])
@pytest.mark.parametrize(
"bounds",
[
{},
{"sigma_min": None, "sigma_max": None},
{"sigma_min": 0.02},
{"sigma_max": 0.7},
],
)
def test_sigma_schedule_conversion_and_steps(cls, schedule, bounds):
scheduler = cls(**{f"use_{schedule}_sigmas": True})
scheduler.register_to_config(**bounds)
reference = EulerDiscreteScheduler()
reference.register_to_config(**bounds)
methods = lambda s: {
"karras": s._convert_to_karras,
"exponential": s._convert_to_exponential,
"beta": s._convert_to_beta,
}
for steps in (1, 7):
sigmas = torch.linspace(0.9, 0.01, 16)
actual = methods(scheduler)[schedule](sigmas, steps)
expected = methods(reference)[schedule](sigmas, steps)
np.testing.assert_array_equal(actual, expected)
outputs = []
for _ in range(2):
scheduler.set_timesteps(7)
assert torch.isfinite(scheduler.sigmas).all()
assert (scheduler.sigmas[1:] <= scheduler.sigmas[:-1]).all()
sample = torch.linspace(-1, 1, 48).reshape(1, 3, 4, 4)
for timestep in scheduler.timesteps:
sample = scheduler.step(sample * 0.1, timestep, sample).prev_sample
assert torch.isfinite(sample).all()
outputs.append(sample)
torch.testing.assert_close(*outputs, rtol=0, atol=0)