From 7dd512b0e879710d1162d657b5eeea4bfd7a4e60 Mon Sep 17 00:00:00 2001 From: Mick Date: Thu, 8 Oct 2026 18:18:39 +0800 Subject: [PATCH] [diffusion] refactor: deduplicate FLUX positions and RoPE application (#43002) Co-authored-by: Mick Qian --- .../runtime/layers/rotary_embedding/base.py | 96 +++++++------------ .../runtime/layers/rotary_embedding/mrope.py | 25 +++++ .../runtime/layers/rotary_embedding/utils.py | 6 +- .../runtime/models/dits/flux.py | 31 +----- .../runtime/models/dits/flux_2.py | 33 +------ 5 files changed, 64 insertions(+), 127 deletions(-) diff --git a/python/sglang/multimodal_gen/runtime/layers/rotary_embedding/base.py b/python/sglang/multimodal_gen/runtime/layers/rotary_embedding/base.py index d89b9936b8..11b0ba75d2 100644 --- a/python/sglang/multimodal_gen/runtime/layers/rotary_embedding/base.py +++ b/python/sglang/multimodal_gen/runtime/layers/rotary_embedding/base.py @@ -85,6 +85,35 @@ class RotaryEmbedding(CustomOp): cache = torch.cat((cos, sin), dim=-1) return cache + def _apply_cos_sin( + self, + query: torch.Tensor, + key: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """rotate Q/K in the caller's layout, preserving broadcast and unrotated tails""" + q_rot = query[..., : self.rotary_dim] + q_pass = query[..., self.rotary_dim :] + k_rot = key[..., : self.rotary_dim] + k_pass = key[..., self.rotary_dim :] + + q_rotated = _apply_rotary_emb( + q_rot, + cos, + sin, + is_neox_style=self.is_neox_style, + interleaved=not self.is_neox_style, + ) + k_rotated = _apply_rotary_emb( + k_rot, + cos, + sin, + is_neox_style=self.is_neox_style, + interleaved=not self.is_neox_style, + ) + return self._combine_rotated_and_pass(q_rotated, k_rotated, q_pass, k_pass) + def _combine_rotated_and_pass( self, q_rotated: torch.Tensor, @@ -216,27 +245,7 @@ class RotaryEmbedding(CustomOp): # No [batch*seq, ...] flatten: cos/sin are [seq_len, # rotary_dim // 2], shared across the batch, and only # broadcast correctly this way for batch_size > 1. - q_rot = query[..., : self.rotary_dim] - q_pass = query[..., self.rotary_dim :] - - k_rot = key[..., : self.rotary_dim] - k_pass = key[..., self.rotary_dim :] - - q_rotated = _apply_rotary_emb( - q_rot, - cos, - sin, - is_neox_style=self.is_neox_style, - interleaved=not self.is_neox_style, - ) - k_rotated = _apply_rotary_emb( - k_rot, - cos, - sin, - is_neox_style=self.is_neox_style, - interleaved=not self.is_neox_style, - ) - return self._combine_rotated_and_pass(q_rotated, k_rotated, q_pass, k_pass) + return self._apply_cos_sin(query, key, cos, sin) if cos_sin_cache is not None: return self.forward_native( @@ -384,55 +393,14 @@ class RotaryEmbedding(CustomOp): # num_tokens = batch*seq row-for-row. q_shape = query.shape q_flat = query.reshape(num_tokens, -1, self.head_size) - q_rot = q_flat[..., : self.rotary_dim] - q_pass = q_flat[..., self.rotary_dim :] - k_shape = key.shape k_flat = key.reshape(num_tokens, -1, self.head_size) - k_rot = k_flat[..., : self.rotary_dim] - k_pass = k_flat[..., self.rotary_dim :] - - q_rotated = _apply_rotary_emb( - q_rot, - cos, - sin, - is_neox_style=self.is_neox_style, - interleaved=not self.is_neox_style, - ) - k_rotated = _apply_rotary_emb( - k_rot, - cos, - sin, - is_neox_style=self.is_neox_style, - interleaved=not self.is_neox_style, - ) - q, k = self._combine_rotated_and_pass( - q_rotated, k_rotated, q_pass, k_pass - ) + q, k = self._apply_cos_sin(q_flat, k_flat, cos, sin) return q.reshape(q_shape), k.reshape(k_shape) # Direct DiT-style call: same batch/seq broadcast reasoning as # forward_npu's cos/sin path. - q_rot = query[..., : self.rotary_dim] - q_pass = query[..., self.rotary_dim :] - k_rot = key[..., : self.rotary_dim] - k_pass = key[..., self.rotary_dim :] - - q_rotated = _apply_rotary_emb( - q_rot, - cos, - sin, - is_neox_style=self.is_neox_style, - interleaved=not self.is_neox_style, - ) - k_rotated = _apply_rotary_emb( - k_rot, - cos, - sin, - is_neox_style=self.is_neox_style, - interleaved=not self.is_neox_style, - ) - return self._combine_rotated_and_pass(q_rotated, k_rotated, q_pass, k_pass) + return self._apply_cos_sin(query, key, cos, sin) if query.dim() != 4 or key.dim() != 4: raise ValueError( diff --git a/python/sglang/multimodal_gen/runtime/layers/rotary_embedding/mrope.py b/python/sglang/multimodal_gen/runtime/layers/rotary_embedding/mrope.py index e284afab84..2981393bf0 100644 --- a/python/sglang/multimodal_gen/runtime/layers/rotary_embedding/mrope.py +++ b/python/sglang/multimodal_gen/runtime/layers/rotary_embedding/mrope.py @@ -5,6 +5,7 @@ import functools import torch from sglang.multimodal_gen.runtime.distributed.parallel_state import get_sp_group +from sglang.multimodal_gen.runtime.platforms import current_platform def _to_tuple(x: int | tuple[int, ...], dim: int = 2) -> tuple[int, ...]: @@ -500,3 +501,27 @@ class NDRotaryEmbedding(torch.nn.Module): col_offset += dim_i_half return cos.float(), sin.float() + + +class FluxPosEmbed(torch.nn.Module): + """uncached FLUX-family RoPE with contiguous float32 outputs""" + + # modified from https://github.com/black-forest-labs/flux/blob/c00d7c60b085fce8058b9df845e036090873f2ce/src/flux/modules/layers.py#L11 + def __init__(self, theta: int, axes_dim: list[int]): + super().__init__() + self.rope = NDRotaryEmbedding( + rope_dim_list=axes_dim, + rope_theta=theta, + use_real=False, + repeat_interleave_real=False, + dtype=( + torch.float64 + if current_platform.is_float64_supported() + else torch.float32 + ), + ) + + def forward(self, ids: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + pos = ids.float() + freqs_cos, freqs_sin = self.rope.forward_uncached(pos=pos) + return freqs_cos.contiguous().float(), freqs_sin.contiguous().float() diff --git a/python/sglang/multimodal_gen/runtime/layers/rotary_embedding/utils.py b/python/sglang/multimodal_gen/runtime/layers/rotary_embedding/utils.py index 0134249d93..b173734fc5 100644 --- a/python/sglang/multimodal_gen/runtime/layers/rotary_embedding/utils.py +++ b/python/sglang/multimodal_gen/runtime/layers/rotary_embedding/utils.py @@ -53,11 +53,7 @@ def _apply_rotary_emb( if is_neox_style: cos = cos.unsqueeze(-2) sin = sin.unsqueeze(-2) - if is_neox_style: - x1, x2 = torch.chunk(x, 2, dim=-1) - else: - x1 = x[..., ::2] - x2 = x[..., 1::2] + x1, x2 = torch.chunk(x, 2, dim=-1) o1 = (x1.float() * cos - x2.float() * sin).type_as(x) o2 = (x2.float() * cos + x1.float() * sin).type_as(x) return torch.cat((o1, o2), dim=-1) diff --git a/python/sglang/multimodal_gen/runtime/models/dits/flux.py b/python/sglang/multimodal_gen/runtime/models/dits/flux.py index 4a4a6ecf58..a5c2e9d353 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/flux.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/flux.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, Optional, Tuple, Union import torch import torch.nn as nn @@ -76,8 +76,8 @@ from sglang.multimodal_gen.runtime.layers.quantization.configs.nunchaku_config i NunchakuConfig, is_nunchaku_available, ) -from sglang.multimodal_gen.runtime.layers.rotary_embedding import ( - NDRotaryEmbedding, +from sglang.multimodal_gen.runtime.layers.rotary_embedding.mrope import ( + FluxPosEmbed, ) from sglang.multimodal_gen.runtime.layers.visual_embedding import ( CombinedTimestepGuidanceTextProjEmbeddings, @@ -91,7 +91,6 @@ from sglang.multimodal_gen.runtime.models.dits.base import CachableDiT from sglang.multimodal_gen.runtime.models.dits.common import get_qkv_projections from sglang.multimodal_gen.runtime.platforms import ( AttentionBackendEnum, - current_platform, ) from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger @@ -1191,30 +1190,6 @@ class FluxTransformerBlock(nn.Module): return encoder_hidden_states, hidden_states -class FluxPosEmbed(nn.Module): - # modified from https://github.com/black-forest-labs/flux/blob/c00d7c60b085fce8058b9df845e036090873f2ce/src/flux/modules/layers.py#L11 - def __init__(self, theta: int, axes_dim: List[int]): - super().__init__() - self.rope = NDRotaryEmbedding( - rope_dim_list=axes_dim, - rope_theta=theta, - use_real=False, - repeat_interleave_real=False, - dtype=( - torch.float64 - if current_platform.is_float64_supported() - else torch.float32 - ), - ) - - def forward(self, ids: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: - pos = ids.float() - # TODO: potential error: flux use n_axes = ids.shape[-1] - # see: https://github.com/huggingface/diffusers/blob/17c0e79dbdf53fb6705e9c09cc1a854b84c39249/src/diffusers/models/transformers/transformer_flux.py#L509 - freqs_cos, freqs_sin = self.rope.forward_uncached(pos=pos) - return freqs_cos.contiguous().float(), freqs_sin.contiguous().float() - - class FluxTransformer2DModel(CachableDiT, LayerwiseOffloadableModuleMixin): """ The Transformer model introduced in Flux. diff --git a/python/sglang/multimodal_gen/runtime/models/dits/flux_2.py b/python/sglang/multimodal_gen/runtime/models/dits/flux_2.py index 7fc9e17801..f839d78bef 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/flux_2.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/flux_2.py @@ -13,7 +13,7 @@ # limitations under the License. import os -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, Optional, Tuple import torch import torch.nn as nn @@ -77,8 +77,8 @@ from sglang.multimodal_gen.runtime.layers.quantization.modelopt_quant import ( apply_nvfp4_gemm_prequantized, apply_nvfp4_gemm_swiglu_quant, ) -from sglang.multimodal_gen.runtime.layers.rotary_embedding import ( - NDRotaryEmbedding, +from sglang.multimodal_gen.runtime.layers.rotary_embedding.mrope import ( + FluxPosEmbed as Flux2PosEmbed, ) from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import ( LayerwiseOffloadableModuleMixin, @@ -1451,33 +1451,6 @@ class Flux2Modulation(nn.Module): ) -class Flux2PosEmbed(nn.Module): - def __init__(self, theta: int, axes_dim: List[int]): - super().__init__() - self.rope = NDRotaryEmbedding( - rope_dim_list=axes_dim, - rope_theta=theta, - use_real=False, - repeat_interleave_real=False, - dtype=( - torch.float64 - if ( - current_platform.is_float64_supported() - if hasattr(current_platform, "is_float64_supported") - else True - ) - else torch.float32 - ), - ) - - def forward(self, ids: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: - pos = ids.float() - # TODO: potential error: flux use n_axes = ids.shape[-1] - # see: https://github.com/huggingface/diffusers/blob/17c0e79dbdf53fb6705e9c09cc1a854b84c39249/src/diffusers/models/transformers/transformer_flux.py#L509 - freqs_cos, freqs_sin = self.rope.forward_uncached(pos=pos) - return freqs_cos.contiguous().float(), freqs_sin.contiguous().float() - - class Flux2Transformer2DModel(CachableDiT, LayerwiseOffloadableModuleMixin): """ The Transformer model introduced in Flux 2.