[diffusion] refactor: deduplicate FLUX positions and RoPE application (#43002)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
Mick
2026-10-08 18:18:39 +08:00
committed by GitHub
co-authored by Mick Qian
parent 8ad24fb560
commit 7dd512b0e8
5 changed files with 64 additions and 127 deletions
@@ -85,6 +85,35 @@ class RotaryEmbedding(CustomOp):
cache = torch.cat((cos, sin), dim=-1)
return cache
def _apply_cos_sin(
self,
query: torch.Tensor,
key: torch.Tensor,
cos: torch.Tensor,
sin: torch.Tensor,
) -> Tuple[torch.Tensor, torch.Tensor]:
"""rotate Q/K in the caller's layout, preserving broadcast and unrotated tails"""
q_rot = query[..., : self.rotary_dim]
q_pass = query[..., self.rotary_dim :]
k_rot = key[..., : self.rotary_dim]
k_pass = key[..., self.rotary_dim :]
q_rotated = _apply_rotary_emb(
q_rot,
cos,
sin,
is_neox_style=self.is_neox_style,
interleaved=not self.is_neox_style,
)
k_rotated = _apply_rotary_emb(
k_rot,
cos,
sin,
is_neox_style=self.is_neox_style,
interleaved=not self.is_neox_style,
)
return self._combine_rotated_and_pass(q_rotated, k_rotated, q_pass, k_pass)
def _combine_rotated_and_pass(
self,
q_rotated: torch.Tensor,
@@ -216,27 +245,7 @@ class RotaryEmbedding(CustomOp):
# No [batch*seq, ...] flatten: cos/sin are [seq_len,
# rotary_dim // 2], shared across the batch, and only
# broadcast correctly this way for batch_size > 1.
q_rot = query[..., : self.rotary_dim]
q_pass = query[..., self.rotary_dim :]
k_rot = key[..., : self.rotary_dim]
k_pass = key[..., self.rotary_dim :]
q_rotated = _apply_rotary_emb(
q_rot,
cos,
sin,
is_neox_style=self.is_neox_style,
interleaved=not self.is_neox_style,
)
k_rotated = _apply_rotary_emb(
k_rot,
cos,
sin,
is_neox_style=self.is_neox_style,
interleaved=not self.is_neox_style,
)
return self._combine_rotated_and_pass(q_rotated, k_rotated, q_pass, k_pass)
return self._apply_cos_sin(query, key, cos, sin)
if cos_sin_cache is not None:
return self.forward_native(
@@ -384,55 +393,14 @@ class RotaryEmbedding(CustomOp):
# num_tokens = batch*seq row-for-row.
q_shape = query.shape
q_flat = query.reshape(num_tokens, -1, self.head_size)
q_rot = q_flat[..., : self.rotary_dim]
q_pass = q_flat[..., self.rotary_dim :]
k_shape = key.shape
k_flat = key.reshape(num_tokens, -1, self.head_size)
k_rot = k_flat[..., : self.rotary_dim]
k_pass = k_flat[..., self.rotary_dim :]
q_rotated = _apply_rotary_emb(
q_rot,
cos,
sin,
is_neox_style=self.is_neox_style,
interleaved=not self.is_neox_style,
)
k_rotated = _apply_rotary_emb(
k_rot,
cos,
sin,
is_neox_style=self.is_neox_style,
interleaved=not self.is_neox_style,
)
q, k = self._combine_rotated_and_pass(
q_rotated, k_rotated, q_pass, k_pass
)
q, k = self._apply_cos_sin(q_flat, k_flat, cos, sin)
return q.reshape(q_shape), k.reshape(k_shape)
# Direct DiT-style call: same batch/seq broadcast reasoning as
# forward_npu's cos/sin path.
q_rot = query[..., : self.rotary_dim]
q_pass = query[..., self.rotary_dim :]
k_rot = key[..., : self.rotary_dim]
k_pass = key[..., self.rotary_dim :]
q_rotated = _apply_rotary_emb(
q_rot,
cos,
sin,
is_neox_style=self.is_neox_style,
interleaved=not self.is_neox_style,
)
k_rotated = _apply_rotary_emb(
k_rot,
cos,
sin,
is_neox_style=self.is_neox_style,
interleaved=not self.is_neox_style,
)
return self._combine_rotated_and_pass(q_rotated, k_rotated, q_pass, k_pass)
return self._apply_cos_sin(query, key, cos, sin)
if query.dim() != 4 or key.dim() != 4:
raise ValueError(
@@ -5,6 +5,7 @@ import functools
import torch
from sglang.multimodal_gen.runtime.distributed.parallel_state import get_sp_group
from sglang.multimodal_gen.runtime.platforms import current_platform
def _to_tuple(x: int | tuple[int, ...], dim: int = 2) -> tuple[int, ...]:
@@ -500,3 +501,27 @@ class NDRotaryEmbedding(torch.nn.Module):
col_offset += dim_i_half
return cos.float(), sin.float()
class FluxPosEmbed(torch.nn.Module):
"""uncached FLUX-family RoPE with contiguous float32 outputs"""
# modified from https://github.com/black-forest-labs/flux/blob/c00d7c60b085fce8058b9df845e036090873f2ce/src/flux/modules/layers.py#L11
def __init__(self, theta: int, axes_dim: list[int]):
super().__init__()
self.rope = NDRotaryEmbedding(
rope_dim_list=axes_dim,
rope_theta=theta,
use_real=False,
repeat_interleave_real=False,
dtype=(
torch.float64
if current_platform.is_float64_supported()
else torch.float32
),
)
def forward(self, ids: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
pos = ids.float()
freqs_cos, freqs_sin = self.rope.forward_uncached(pos=pos)
return freqs_cos.contiguous().float(), freqs_sin.contiguous().float()
@@ -53,11 +53,7 @@ def _apply_rotary_emb(
if is_neox_style:
cos = cos.unsqueeze(-2)
sin = sin.unsqueeze(-2)
if is_neox_style:
x1, x2 = torch.chunk(x, 2, dim=-1)
else:
x1 = x[..., ::2]
x2 = x[..., 1::2]
x1, x2 = torch.chunk(x, 2, dim=-1)
o1 = (x1.float() * cos - x2.float() * sin).type_as(x)
o2 = (x2.float() * cos + x1.float() * sin).type_as(x)
return torch.cat((o1, o2), dim=-1)
@@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import Any, Dict, List, Optional, Tuple, Union
from typing import Any, Dict, Optional, Tuple, Union
import torch
import torch.nn as nn
@@ -76,8 +76,8 @@ from sglang.multimodal_gen.runtime.layers.quantization.configs.nunchaku_config i
NunchakuConfig,
is_nunchaku_available,
)
from sglang.multimodal_gen.runtime.layers.rotary_embedding import (
NDRotaryEmbedding,
from sglang.multimodal_gen.runtime.layers.rotary_embedding.mrope import (
FluxPosEmbed,
)
from sglang.multimodal_gen.runtime.layers.visual_embedding import (
CombinedTimestepGuidanceTextProjEmbeddings,
@@ -91,7 +91,6 @@ from sglang.multimodal_gen.runtime.models.dits.base import CachableDiT
from sglang.multimodal_gen.runtime.models.dits.common import get_qkv_projections
from sglang.multimodal_gen.runtime.platforms import (
AttentionBackendEnum,
current_platform,
)
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
@@ -1191,30 +1190,6 @@ class FluxTransformerBlock(nn.Module):
return encoder_hidden_states, hidden_states
class FluxPosEmbed(nn.Module):
# modified from https://github.com/black-forest-labs/flux/blob/c00d7c60b085fce8058b9df845e036090873f2ce/src/flux/modules/layers.py#L11
def __init__(self, theta: int, axes_dim: List[int]):
super().__init__()
self.rope = NDRotaryEmbedding(
rope_dim_list=axes_dim,
rope_theta=theta,
use_real=False,
repeat_interleave_real=False,
dtype=(
torch.float64
if current_platform.is_float64_supported()
else torch.float32
),
)
def forward(self, ids: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
pos = ids.float()
# TODO: potential error: flux use n_axes = ids.shape[-1]
# see: https://github.com/huggingface/diffusers/blob/17c0e79dbdf53fb6705e9c09cc1a854b84c39249/src/diffusers/models/transformers/transformer_flux.py#L509
freqs_cos, freqs_sin = self.rope.forward_uncached(pos=pos)
return freqs_cos.contiguous().float(), freqs_sin.contiguous().float()
class FluxTransformer2DModel(CachableDiT, LayerwiseOffloadableModuleMixin):
"""
The Transformer model introduced in Flux.
@@ -13,7 +13,7 @@
# limitations under the License.
import os
from typing import Any, Dict, List, Optional, Tuple
from typing import Any, Dict, Optional, Tuple
import torch
import torch.nn as nn
@@ -77,8 +77,8 @@ from sglang.multimodal_gen.runtime.layers.quantization.modelopt_quant import (
apply_nvfp4_gemm_prequantized,
apply_nvfp4_gemm_swiglu_quant,
)
from sglang.multimodal_gen.runtime.layers.rotary_embedding import (
NDRotaryEmbedding,
from sglang.multimodal_gen.runtime.layers.rotary_embedding.mrope import (
FluxPosEmbed as Flux2PosEmbed,
)
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
LayerwiseOffloadableModuleMixin,
@@ -1451,33 +1451,6 @@ class Flux2Modulation(nn.Module):
)
class Flux2PosEmbed(nn.Module):
def __init__(self, theta: int, axes_dim: List[int]):
super().__init__()
self.rope = NDRotaryEmbedding(
rope_dim_list=axes_dim,
rope_theta=theta,
use_real=False,
repeat_interleave_real=False,
dtype=(
torch.float64
if (
current_platform.is_float64_supported()
if hasattr(current_platform, "is_float64_supported")
else True
)
else torch.float32
),
)
def forward(self, ids: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
pos = ids.float()
# TODO: potential error: flux use n_axes = ids.shape[-1]
# see: https://github.com/huggingface/diffusers/blob/17c0e79dbdf53fb6705e9c09cc1a854b84c39249/src/diffusers/models/transformers/transformer_flux.py#L509
freqs_cos, freqs_sin = self.rope.forward_uncached(pos=pos)
return freqs_cos.contiguous().float(), freqs_sin.contiguous().float()
class Flux2Transformer2DModel(CachableDiT, LayerwiseOffloadableModuleMixin):
"""
The Transformer model introduced in Flux 2.