[diffusion] refactor: deduplicate FLUX positions and RoPE application (#43002)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
@@ -85,6 +85,35 @@ class RotaryEmbedding(CustomOp):
|
||||
cache = torch.cat((cos, sin), dim=-1)
|
||||
return cache
|
||||
|
||||
def _apply_cos_sin(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
cos: torch.Tensor,
|
||||
sin: torch.Tensor,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""rotate Q/K in the caller's layout, preserving broadcast and unrotated tails"""
|
||||
q_rot = query[..., : self.rotary_dim]
|
||||
q_pass = query[..., self.rotary_dim :]
|
||||
k_rot = key[..., : self.rotary_dim]
|
||||
k_pass = key[..., self.rotary_dim :]
|
||||
|
||||
q_rotated = _apply_rotary_emb(
|
||||
q_rot,
|
||||
cos,
|
||||
sin,
|
||||
is_neox_style=self.is_neox_style,
|
||||
interleaved=not self.is_neox_style,
|
||||
)
|
||||
k_rotated = _apply_rotary_emb(
|
||||
k_rot,
|
||||
cos,
|
||||
sin,
|
||||
is_neox_style=self.is_neox_style,
|
||||
interleaved=not self.is_neox_style,
|
||||
)
|
||||
return self._combine_rotated_and_pass(q_rotated, k_rotated, q_pass, k_pass)
|
||||
|
||||
def _combine_rotated_and_pass(
|
||||
self,
|
||||
q_rotated: torch.Tensor,
|
||||
@@ -216,27 +245,7 @@ class RotaryEmbedding(CustomOp):
|
||||
# No [batch*seq, ...] flatten: cos/sin are [seq_len,
|
||||
# rotary_dim // 2], shared across the batch, and only
|
||||
# broadcast correctly this way for batch_size > 1.
|
||||
q_rot = query[..., : self.rotary_dim]
|
||||
q_pass = query[..., self.rotary_dim :]
|
||||
|
||||
k_rot = key[..., : self.rotary_dim]
|
||||
k_pass = key[..., self.rotary_dim :]
|
||||
|
||||
q_rotated = _apply_rotary_emb(
|
||||
q_rot,
|
||||
cos,
|
||||
sin,
|
||||
is_neox_style=self.is_neox_style,
|
||||
interleaved=not self.is_neox_style,
|
||||
)
|
||||
k_rotated = _apply_rotary_emb(
|
||||
k_rot,
|
||||
cos,
|
||||
sin,
|
||||
is_neox_style=self.is_neox_style,
|
||||
interleaved=not self.is_neox_style,
|
||||
)
|
||||
return self._combine_rotated_and_pass(q_rotated, k_rotated, q_pass, k_pass)
|
||||
return self._apply_cos_sin(query, key, cos, sin)
|
||||
|
||||
if cos_sin_cache is not None:
|
||||
return self.forward_native(
|
||||
@@ -384,55 +393,14 @@ class RotaryEmbedding(CustomOp):
|
||||
# num_tokens = batch*seq row-for-row.
|
||||
q_shape = query.shape
|
||||
q_flat = query.reshape(num_tokens, -1, self.head_size)
|
||||
q_rot = q_flat[..., : self.rotary_dim]
|
||||
q_pass = q_flat[..., self.rotary_dim :]
|
||||
|
||||
k_shape = key.shape
|
||||
k_flat = key.reshape(num_tokens, -1, self.head_size)
|
||||
k_rot = k_flat[..., : self.rotary_dim]
|
||||
k_pass = k_flat[..., self.rotary_dim :]
|
||||
|
||||
q_rotated = _apply_rotary_emb(
|
||||
q_rot,
|
||||
cos,
|
||||
sin,
|
||||
is_neox_style=self.is_neox_style,
|
||||
interleaved=not self.is_neox_style,
|
||||
)
|
||||
k_rotated = _apply_rotary_emb(
|
||||
k_rot,
|
||||
cos,
|
||||
sin,
|
||||
is_neox_style=self.is_neox_style,
|
||||
interleaved=not self.is_neox_style,
|
||||
)
|
||||
q, k = self._combine_rotated_and_pass(
|
||||
q_rotated, k_rotated, q_pass, k_pass
|
||||
)
|
||||
q, k = self._apply_cos_sin(q_flat, k_flat, cos, sin)
|
||||
return q.reshape(q_shape), k.reshape(k_shape)
|
||||
|
||||
# Direct DiT-style call: same batch/seq broadcast reasoning as
|
||||
# forward_npu's cos/sin path.
|
||||
q_rot = query[..., : self.rotary_dim]
|
||||
q_pass = query[..., self.rotary_dim :]
|
||||
k_rot = key[..., : self.rotary_dim]
|
||||
k_pass = key[..., self.rotary_dim :]
|
||||
|
||||
q_rotated = _apply_rotary_emb(
|
||||
q_rot,
|
||||
cos,
|
||||
sin,
|
||||
is_neox_style=self.is_neox_style,
|
||||
interleaved=not self.is_neox_style,
|
||||
)
|
||||
k_rotated = _apply_rotary_emb(
|
||||
k_rot,
|
||||
cos,
|
||||
sin,
|
||||
is_neox_style=self.is_neox_style,
|
||||
interleaved=not self.is_neox_style,
|
||||
)
|
||||
return self._combine_rotated_and_pass(q_rotated, k_rotated, q_pass, k_pass)
|
||||
return self._apply_cos_sin(query, key, cos, sin)
|
||||
|
||||
if query.dim() != 4 or key.dim() != 4:
|
||||
raise ValueError(
|
||||
|
||||
@@ -5,6 +5,7 @@ import functools
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import get_sp_group
|
||||
from sglang.multimodal_gen.runtime.platforms import current_platform
|
||||
|
||||
|
||||
def _to_tuple(x: int | tuple[int, ...], dim: int = 2) -> tuple[int, ...]:
|
||||
@@ -500,3 +501,27 @@ class NDRotaryEmbedding(torch.nn.Module):
|
||||
col_offset += dim_i_half
|
||||
|
||||
return cos.float(), sin.float()
|
||||
|
||||
|
||||
class FluxPosEmbed(torch.nn.Module):
|
||||
"""uncached FLUX-family RoPE with contiguous float32 outputs"""
|
||||
|
||||
# modified from https://github.com/black-forest-labs/flux/blob/c00d7c60b085fce8058b9df845e036090873f2ce/src/flux/modules/layers.py#L11
|
||||
def __init__(self, theta: int, axes_dim: list[int]):
|
||||
super().__init__()
|
||||
self.rope = NDRotaryEmbedding(
|
||||
rope_dim_list=axes_dim,
|
||||
rope_theta=theta,
|
||||
use_real=False,
|
||||
repeat_interleave_real=False,
|
||||
dtype=(
|
||||
torch.float64
|
||||
if current_platform.is_float64_supported()
|
||||
else torch.float32
|
||||
),
|
||||
)
|
||||
|
||||
def forward(self, ids: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
pos = ids.float()
|
||||
freqs_cos, freqs_sin = self.rope.forward_uncached(pos=pos)
|
||||
return freqs_cos.contiguous().float(), freqs_sin.contiguous().float()
|
||||
|
||||
@@ -53,11 +53,7 @@ def _apply_rotary_emb(
|
||||
if is_neox_style:
|
||||
cos = cos.unsqueeze(-2)
|
||||
sin = sin.unsqueeze(-2)
|
||||
if is_neox_style:
|
||||
x1, x2 = torch.chunk(x, 2, dim=-1)
|
||||
else:
|
||||
x1 = x[..., ::2]
|
||||
x2 = x[..., 1::2]
|
||||
x1, x2 = torch.chunk(x, 2, dim=-1)
|
||||
o1 = (x1.float() * cos - x2.float() * sin).type_as(x)
|
||||
o2 = (x2.float() * cos + x1.float() * sin).type_as(x)
|
||||
return torch.cat((o1, o2), dim=-1)
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from typing import Any, Dict, List, Optional, Tuple, Union
|
||||
from typing import Any, Dict, Optional, Tuple, Union
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
@@ -76,8 +76,8 @@ from sglang.multimodal_gen.runtime.layers.quantization.configs.nunchaku_config i
|
||||
NunchakuConfig,
|
||||
is_nunchaku_available,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.rotary_embedding import (
|
||||
NDRotaryEmbedding,
|
||||
from sglang.multimodal_gen.runtime.layers.rotary_embedding.mrope import (
|
||||
FluxPosEmbed,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.visual_embedding import (
|
||||
CombinedTimestepGuidanceTextProjEmbeddings,
|
||||
@@ -91,7 +91,6 @@ from sglang.multimodal_gen.runtime.models.dits.base import CachableDiT
|
||||
from sglang.multimodal_gen.runtime.models.dits.common import get_qkv_projections
|
||||
from sglang.multimodal_gen.runtime.platforms import (
|
||||
AttentionBackendEnum,
|
||||
current_platform,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
|
||||
@@ -1191,30 +1190,6 @@ class FluxTransformerBlock(nn.Module):
|
||||
return encoder_hidden_states, hidden_states
|
||||
|
||||
|
||||
class FluxPosEmbed(nn.Module):
|
||||
# modified from https://github.com/black-forest-labs/flux/blob/c00d7c60b085fce8058b9df845e036090873f2ce/src/flux/modules/layers.py#L11
|
||||
def __init__(self, theta: int, axes_dim: List[int]):
|
||||
super().__init__()
|
||||
self.rope = NDRotaryEmbedding(
|
||||
rope_dim_list=axes_dim,
|
||||
rope_theta=theta,
|
||||
use_real=False,
|
||||
repeat_interleave_real=False,
|
||||
dtype=(
|
||||
torch.float64
|
||||
if current_platform.is_float64_supported()
|
||||
else torch.float32
|
||||
),
|
||||
)
|
||||
|
||||
def forward(self, ids: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
pos = ids.float()
|
||||
# TODO: potential error: flux use n_axes = ids.shape[-1]
|
||||
# see: https://github.com/huggingface/diffusers/blob/17c0e79dbdf53fb6705e9c09cc1a854b84c39249/src/diffusers/models/transformers/transformer_flux.py#L509
|
||||
freqs_cos, freqs_sin = self.rope.forward_uncached(pos=pos)
|
||||
return freqs_cos.contiguous().float(), freqs_sin.contiguous().float()
|
||||
|
||||
|
||||
class FluxTransformer2DModel(CachableDiT, LayerwiseOffloadableModuleMixin):
|
||||
"""
|
||||
The Transformer model introduced in Flux.
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
@@ -77,8 +77,8 @@ from sglang.multimodal_gen.runtime.layers.quantization.modelopt_quant import (
|
||||
apply_nvfp4_gemm_prequantized,
|
||||
apply_nvfp4_gemm_swiglu_quant,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.rotary_embedding import (
|
||||
NDRotaryEmbedding,
|
||||
from sglang.multimodal_gen.runtime.layers.rotary_embedding.mrope import (
|
||||
FluxPosEmbed as Flux2PosEmbed,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
|
||||
LayerwiseOffloadableModuleMixin,
|
||||
@@ -1451,33 +1451,6 @@ class Flux2Modulation(nn.Module):
|
||||
)
|
||||
|
||||
|
||||
class Flux2PosEmbed(nn.Module):
|
||||
def __init__(self, theta: int, axes_dim: List[int]):
|
||||
super().__init__()
|
||||
self.rope = NDRotaryEmbedding(
|
||||
rope_dim_list=axes_dim,
|
||||
rope_theta=theta,
|
||||
use_real=False,
|
||||
repeat_interleave_real=False,
|
||||
dtype=(
|
||||
torch.float64
|
||||
if (
|
||||
current_platform.is_float64_supported()
|
||||
if hasattr(current_platform, "is_float64_supported")
|
||||
else True
|
||||
)
|
||||
else torch.float32
|
||||
),
|
||||
)
|
||||
|
||||
def forward(self, ids: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
pos = ids.float()
|
||||
# TODO: potential error: flux use n_axes = ids.shape[-1]
|
||||
# see: https://github.com/huggingface/diffusers/blob/17c0e79dbdf53fb6705e9c09cc1a854b84c39249/src/diffusers/models/transformers/transformer_flux.py#L509
|
||||
freqs_cos, freqs_sin = self.rope.forward_uncached(pos=pos)
|
||||
return freqs_cos.contiguous().float(), freqs_sin.contiguous().float()
|
||||
|
||||
|
||||
class Flux2Transformer2DModel(CachableDiT, LayerwiseOffloadableModuleMixin):
|
||||
"""
|
||||
The Transformer model introduced in Flux 2.
|
||||
|
||||
Reference in New Issue
Block a user