[Perf] [NVIDIA] diffusion/FLUX.3: Run dynamic LoRA through PrimsTS GEMMs (#44761)
Co-authored-by: Mick <mickjagger19@icloud.com>
This commit is contained in:
@@ -9,6 +9,7 @@ import torch
|
||||
from flashinfer.prims_ts import gemm as prims_ts_gemm
|
||||
|
||||
from sglang.kernels.ops.diffusion.gemm.flux3_prims_ts_config import (
|
||||
delta_kwargs,
|
||||
flux3_prims_ts_config,
|
||||
)
|
||||
from sglang.kernels.ops.quantization.per_token_quant_fp8 import per_token_quant_fp8
|
||||
@@ -90,6 +91,8 @@ def flux3_fused_mlp_up(
|
||||
packed_weight: torch.Tensor,
|
||||
packed_weight_scale: torch.Tensor,
|
||||
quantized: QuantizedActivation | None = None,
|
||||
*,
|
||||
delta: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Run fused FP8 GEMM + SwiGLU, quantizing BF16 rows when needed."""
|
||||
if x.dtype != torch.bfloat16:
|
||||
@@ -103,6 +106,7 @@ def flux3_fused_mlp_up(
|
||||
config=_flux3_gemm_config(
|
||||
x, epilogue="swiglu", k=packed_weight.shape[1], n=packed_weight.shape[0]
|
||||
),
|
||||
**delta_kwargs(delta),
|
||||
skip_check=True,
|
||||
)
|
||||
return out.view(*x.shape[:-1], out.shape[-1])
|
||||
@@ -114,6 +118,8 @@ def flux3_clustered_fp8_linear(
|
||||
packed_weight_scale: torch.Tensor,
|
||||
quantized: QuantizedActivation | None = None,
|
||||
qkv_scale: torch.Tensor | None = None,
|
||||
*,
|
||||
delta: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Run a PrimsTS FP8 GEMM, optionally scaling packed Q/K/V thirds."""
|
||||
if x.dtype != torch.bfloat16:
|
||||
@@ -132,6 +138,7 @@ def flux3_clustered_fp8_linear(
|
||||
n=packed_weight.shape[0],
|
||||
qkv_scale=qkv_scale is not None,
|
||||
),
|
||||
**delta_kwargs(delta),
|
||||
skip_check=True,
|
||||
)
|
||||
return out.view(*x.shape[:-1], out.shape[-1])
|
||||
@@ -149,6 +156,7 @@ def flux3_clustered_fp8_qkv_qknorm_rope(
|
||||
qkv_scale: torch.Tensor | None = None,
|
||||
*,
|
||||
separate_qkv_output: bool = False,
|
||||
delta: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Fuse QKV GEMM, QK RMSNorm, and RoPE, quantizing rows when needed."""
|
||||
if x.dtype != torch.bfloat16:
|
||||
@@ -194,6 +202,7 @@ def flux3_clustered_fp8_qkv_qknorm_rope(
|
||||
head_dim=q_norm_weight.numel(),
|
||||
qkv_scale=qkv_scale,
|
||||
config=config,
|
||||
**delta_kwargs(delta),
|
||||
skip_check=True,
|
||||
)
|
||||
if separate_qkv_output:
|
||||
|
||||
@@ -16,6 +16,7 @@ from flashinfer import SfLayout, nvfp4_quantize
|
||||
from flashinfer.prims_ts import gemm as prims_ts_gemm
|
||||
|
||||
from sglang.kernels.ops.diffusion.gemm.flux3_prims_ts_config import (
|
||||
delta_kwargs,
|
||||
flux3_prims_ts_config,
|
||||
)
|
||||
|
||||
@@ -253,6 +254,8 @@ def flux3_nvfp4_linear(
|
||||
input_global_scale: float,
|
||||
weight_global_scale: float,
|
||||
quantized: QuantizedNvfp4Activation | None = None,
|
||||
*,
|
||||
delta: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Run a FLUX.3 projection entirely through PrimsTS NVFP4 GEMM."""
|
||||
a_packed, a_scale_128x4, _ = _quantized_activation(x, input_global_scale, quantized)
|
||||
@@ -270,7 +273,7 @@ def flux3_nvfp4_linear(
|
||||
),
|
||||
)
|
||||
if prepared is not None:
|
||||
out = prepared(a_packed, a_scale_128x4)
|
||||
out = prepared(a_packed, a_scale_128x4, **delta_kwargs(delta))
|
||||
else:
|
||||
out = prims_ts_gemm.fp4_linear(
|
||||
a_packed,
|
||||
@@ -279,6 +282,7 @@ def flux3_nvfp4_linear(
|
||||
weight_packed,
|
||||
weight_scale_128x4,
|
||||
weight_global_scale,
|
||||
**delta_kwargs(delta),
|
||||
skip_check=True,
|
||||
)
|
||||
return out.view(*x.shape[:-1], out.shape[-1])
|
||||
@@ -328,6 +332,8 @@ def flux3_nvfp4_linear_swiglu(
|
||||
input_global_scale: float,
|
||||
weight_global_scale: float,
|
||||
quantized: QuantizedNvfp4Activation | None = None,
|
||||
*,
|
||||
delta: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Run FLUX.3's gated MLP-up projection with PrimsTS' SwiGLU epilogue."""
|
||||
a_packed, a_scale_128x4, _ = _quantized_activation(x, input_global_scale, quantized)
|
||||
@@ -347,7 +353,7 @@ def flux3_nvfp4_linear_swiglu(
|
||||
device=x.device,
|
||||
)
|
||||
if prepared is not None:
|
||||
out = prepared(a_packed, a_scale_128x4)
|
||||
out = prepared(a_packed, a_scale_128x4, **delta_kwargs(delta))
|
||||
else:
|
||||
out = prims_ts_gemm.fp4_linear_swiglu(
|
||||
a_packed,
|
||||
@@ -356,6 +362,7 @@ def flux3_nvfp4_linear_swiglu(
|
||||
weight_packed,
|
||||
weight_scale_128x4,
|
||||
weight_global_scale,
|
||||
**delta_kwargs(delta),
|
||||
skip_check=True,
|
||||
)
|
||||
return out.view(*x.shape[:-1], out.shape[-1])
|
||||
@@ -425,6 +432,7 @@ def flux3_nvfp4_qkv_qknorm_rope(
|
||||
quantized: QuantizedNvfp4Activation | None = None,
|
||||
*,
|
||||
separate_qkv_output: bool = False,
|
||||
delta: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Fuse NVFP4 QKV GEMM, Q/K RMSNorm, and RoPE through PrimsTS.
|
||||
|
||||
@@ -458,6 +466,7 @@ def flux3_nvfp4_qkv_qknorm_rope(
|
||||
a_scale_128x4,
|
||||
cos_sin=cos_sin_cache,
|
||||
positions=positions,
|
||||
**delta_kwargs(delta),
|
||||
)
|
||||
elif separate_qkv_output:
|
||||
raise RuntimeError("separate QKV output requires a prepared CUDA projection")
|
||||
@@ -479,6 +488,7 @@ def flux3_nvfp4_qkv_qknorm_rope(
|
||||
head_dim=head_dim,
|
||||
is_neox=False,
|
||||
qkv_scale=qkv_scale,
|
||||
**delta_kwargs(delta),
|
||||
skip_check=True,
|
||||
)
|
||||
if separate_qkv_output:
|
||||
|
||||
@@ -94,6 +94,11 @@ _TACTICS = {
|
||||
}
|
||||
|
||||
|
||||
def delta_kwargs(delta: torch.Tensor | None) -> dict[str, torch.Tensor]:
|
||||
# FlashInfer releases without the PrimsTS delta input reject the keyword.
|
||||
return {} if delta is None else {"delta": delta}
|
||||
|
||||
|
||||
@functools.cache
|
||||
def _tmem_overlap_epilogues() -> frozenset[str]:
|
||||
# FlashInfer PR 6255 (SwiGLU) and 6286 (QKV) add tmem_overlap to these prepare
|
||||
|
||||
@@ -8,6 +8,8 @@ plus SGLang's fused QK-norm/RoPE and SwiGLU kernels, on any device and format.
|
||||
FLUX.3 quant method (:mod:`...quantization.flux3_prims_ts`) prepared at load
|
||||
time. FlashInfer is selected per call, only on SM10x and only for layers whose
|
||||
method planned the matching role, so call sites never branch on the backend.
|
||||
A LoRA-wrapped layer runs its base layer's PrimsTS epilogue with the adapter's
|
||||
delta added before the activation, QK-norm, or output conversion.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -18,15 +20,35 @@ from torch import nn
|
||||
from sglang.kernels.fused_op import BaseFusedOp, register_fused_op
|
||||
from sglang.kernels.spec import FormatSignature, KernelBackend
|
||||
from sglang.multimodal_gen.runtime.layers.layernorm import apply_qk_norm_rope
|
||||
from sglang.multimodal_gen.runtime.layers.lora.linear import BaseLayerWithLoRA
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.flux3_prims_ts import (
|
||||
FLUX3_PRIMS_TS_CAPABILITIES,
|
||||
Flux3PrimsTsRole,
|
||||
)
|
||||
|
||||
|
||||
def _base_layer(layer: nn.Module) -> nn.Module:
|
||||
"""The quantized layer under an optional LoRA wrapper."""
|
||||
return layer.base_layer if isinstance(layer, BaseLayerWithLoRA) else layer
|
||||
|
||||
|
||||
def _lora_active(layer: nn.Module) -> bool:
|
||||
return isinstance(layer, BaseLayerWithLoRA) and layer.lora_active
|
||||
|
||||
|
||||
def _lora_delta_kwargs(
|
||||
layer: nn.Module, x: torch.Tensor, *, swiglu_rows: bool = False
|
||||
) -> dict[str, torch.Tensor]:
|
||||
"""The ``delta`` kwarg while the LoRA adds something; old FlashInfer rejects it."""
|
||||
if not _lora_active(layer):
|
||||
return {}
|
||||
delta = layer.fused_lora_delta(x, swiglu_rows=swiglu_rows)
|
||||
return {} if delta is None else {"delta": delta}
|
||||
|
||||
|
||||
def _prims_ts_method(layer: nn.Module, role: Flux3PrimsTsRole):
|
||||
"""The layer's quant method if it planned ``role`` for PrimsTS, else None."""
|
||||
method = getattr(layer, "quant_method", None)
|
||||
method = getattr(_base_layer(layer), "quant_method", None)
|
||||
if getattr(method, "prims_ts_role", None) is role:
|
||||
return method
|
||||
return None
|
||||
@@ -93,7 +115,10 @@ class Flux3LinearOp(_Flux3PrimsTsOp):
|
||||
return out
|
||||
|
||||
def forward_flashinfer(self, layer, x, quantized=None) -> torch.Tensor:
|
||||
return layer.quant_method.apply_prims_ts_linear(layer, x, quantized)
|
||||
base = _base_layer(layer)
|
||||
return base.quant_method.apply_prims_ts_linear(
|
||||
base, x, quantized, **_lora_delta_kwargs(layer, x)
|
||||
)
|
||||
|
||||
|
||||
class Flux3QKVQKNormRopeOp(_Flux3PrimsTsOp):
|
||||
@@ -154,14 +179,16 @@ class Flux3QKVQKNormRopeOp(_Flux3PrimsTsOp):
|
||||
local_heads,
|
||||
quantized=None,
|
||||
):
|
||||
qkv = layer.quant_method.apply_prims_ts_qkv_qknorm_rope(
|
||||
layer,
|
||||
base = _base_layer(layer)
|
||||
qkv = base.quant_method.apply_prims_ts_qkv_qknorm_rope(
|
||||
base,
|
||||
x,
|
||||
norm.query_norm.weight,
|
||||
norm.key_norm.weight,
|
||||
cos_sin_cache,
|
||||
positions,
|
||||
quantized,
|
||||
**_lora_delta_kwargs(layer, x),
|
||||
)
|
||||
return _unpack_qkv(qkv, local_heads)
|
||||
|
||||
@@ -185,17 +212,25 @@ class Flux3GatedMLPOp(_Flux3PrimsTsOp):
|
||||
return flux3_linear(mlp_out, _silu_and_mul(mlp))
|
||||
|
||||
def forward_flashinfer(self, mlp_in, mlp_out, x, quantized=None) -> torch.Tensor:
|
||||
up = mlp_in.quant_method
|
||||
up_layer = _base_layer(mlp_in)
|
||||
up = up_layer.quant_method
|
||||
down = _prims_ts_method(mlp_out, Flux3PrimsTsRole.LINEAR)
|
||||
up_delta = _lora_delta_kwargs(mlp_in, x, swiglu_rows=True)
|
||||
if (
|
||||
down is not None
|
||||
and not up_delta
|
||||
and not _lora_active(mlp_out)
|
||||
and up.packed_activation_format is not None
|
||||
and down.packed_activation_format == up.packed_activation_format
|
||||
):
|
||||
# Hand MLP-down the packed SwiGLU output without a BF16 round trip.
|
||||
packed = up.apply_prims_ts_swiglu(mlp_in, x, quantized, packed_output=True)
|
||||
return down.apply_prims_ts_linear_from_packed(mlp_out, packed)
|
||||
return flux3_linear(mlp_out, up.apply_prims_ts_swiglu(mlp_in, x, quantized))
|
||||
packed = up.apply_prims_ts_swiglu(
|
||||
up_layer, x, quantized, packed_output=True
|
||||
)
|
||||
return down.apply_prims_ts_linear_from_packed(_base_layer(mlp_out), packed)
|
||||
# A LoRA on MLP-down needs the BF16 SwiGLU output for its own delta.
|
||||
hidden = up.apply_prims_ts_swiglu(up_layer, x, quantized, **up_delta)
|
||||
return flux3_linear(mlp_out, hidden)
|
||||
|
||||
|
||||
_FLUX3_LINEAR = register_fused_op(Flux3LinearOp(), __name__, "_FLUX3_LINEAR")
|
||||
@@ -241,11 +276,11 @@ def flux3_shared_activation(x: torch.Tensor, qkv: nn.Module, mlp_in: nn.Module):
|
||||
if not x.is_cuda:
|
||||
return None
|
||||
up = _prims_ts_method(mlp_in, Flux3PrimsTsRole.SWIGLU)
|
||||
qkv_method = getattr(qkv, "quant_method", None)
|
||||
qkv_method = getattr(_base_layer(qkv), "quant_method", None)
|
||||
if (
|
||||
up is None
|
||||
or getattr(qkv_method, "prims_ts_role", None) is None
|
||||
or qkv_method.activation_format != up.activation_format
|
||||
):
|
||||
return None
|
||||
return up.quantize_prims_ts_activation(mlp_in, x)
|
||||
return up.quantize_prims_ts_activation(_base_layer(mlp_in), x)
|
||||
|
||||
@@ -180,6 +180,7 @@ class BaseLayerWithLoRA(nn.Module):
|
||||
self.lora_B = None
|
||||
self.lora_output_offset = None
|
||||
self.has_lora_output_offset = False
|
||||
self._fused_lora_cache: tuple[tuple, torch.Tensor, torch.Tensor] | None = None
|
||||
|
||||
@property
|
||||
def weight(self):
|
||||
@@ -276,6 +277,86 @@ class BaseLayerWithLoRA(nn.Module):
|
||||
def slice_lora_b_weights(self, B: torch.Tensor) -> torch.Tensor:
|
||||
return B
|
||||
|
||||
@property
|
||||
def lora_active(self) -> bool:
|
||||
"""Whether ``forward`` adds anything to the base layer's output."""
|
||||
if self.disable_lora:
|
||||
return False
|
||||
if self.merged:
|
||||
return self.has_lora_output_offset
|
||||
return self.lora_A is not None
|
||||
|
||||
@staticmethod
|
||||
def _interleave_swiglu_rows(t: torch.Tensor) -> torch.Tensor:
|
||||
# [gate | value] rows -> [value_0, gate_0, value_1, gate_1, ...]
|
||||
half = t.shape[0] // 2
|
||||
return torch.stack((t[half:], t[:half]), dim=1).reshape(t.shape)
|
||||
|
||||
def _fused_lora_factors(
|
||||
self, device: torch.device, swiglu_rows: bool
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
lora_A, lora_B = self.lora_A, self.lora_B
|
||||
if isinstance(lora_B, DTensor):
|
||||
lora_A, lora_B = lora_A.to_local(), lora_B.to_local()
|
||||
key = (
|
||||
lora_A.data_ptr(),
|
||||
lora_A._version,
|
||||
lora_B.data_ptr(),
|
||||
lora_B._version,
|
||||
swiglu_rows,
|
||||
device,
|
||||
)
|
||||
cached = self._fused_lora_cache
|
||||
if cached is not None and cached[0] == key:
|
||||
return cached[1], cached[2]
|
||||
A = self.slice_lora_a_weights(lora_A.to(device))
|
||||
B = self.slice_lora_b_weights(lora_B.to(device))
|
||||
if A.dim() == 3:
|
||||
# Stacked groups: one shrink GEMM, then a block-diagonal expand.
|
||||
A = A.reshape(-1, A.shape[-1])
|
||||
B = torch.block_diag(*B.unbind(0))
|
||||
if swiglu_rows:
|
||||
B = self._interleave_swiglu_rows(B)
|
||||
A, B = A.contiguous(), B.contiguous()
|
||||
self._fused_lora_cache = (key, A, B)
|
||||
return A, B
|
||||
|
||||
def fused_lora_delta(
|
||||
self, x: torch.Tensor, *, swiglu_rows: bool = False
|
||||
) -> torch.Tensor | None:
|
||||
"""``forward(x) - base_layer(x)`` as a contiguous BF16 ``[tokens, out]``.
|
||||
|
||||
For GEMM epilogues that add the adapter's update themselves. None when
|
||||
there is nothing to add. ``swiglu_rows`` orders the columns like the
|
||||
interleaved FLUX.3 PrimsTS SwiGLU weight (``[value_i, gate_i]`` pairs).
|
||||
"""
|
||||
if not self.lora_active:
|
||||
return None
|
||||
runtime_lora_scale = 1.0 if self.merged else self._runtime_lora_scale()
|
||||
if runtime_lora_scale == 0.0:
|
||||
return None
|
||||
x_2d = x.reshape(-1, x.shape[-1])
|
||||
delta = None
|
||||
if not self.merged:
|
||||
A, B = self._fused_lora_factors(device=x.device, swiglu_rows=swiglu_rows)
|
||||
scale = self.strength * runtime_lora_scale
|
||||
if self.lora_alpha != self.lora_rank:
|
||||
scale *= self.lora_alpha / self.lora_rank # type: ignore
|
||||
hidden = x_2d.to(dtype=A.dtype) @ A.T
|
||||
if scale != 1.0:
|
||||
hidden = hidden * scale
|
||||
delta = hidden @ B.T
|
||||
offset = self._active_lora_output_offset()
|
||||
if offset is not None:
|
||||
offset = offset.to(device=x.device) * runtime_lora_scale
|
||||
if swiglu_rows:
|
||||
offset = self._interleave_swiglu_rows(offset)
|
||||
if delta is None:
|
||||
delta = offset.expand(x_2d.shape[0], -1)
|
||||
else:
|
||||
delta = delta + offset.to(dtype=delta.dtype)
|
||||
return delta.to(torch.bfloat16).contiguous()
|
||||
|
||||
def _scaled_lora_output_offset(
|
||||
self,
|
||||
offset: torch.Tensor | None,
|
||||
|
||||
@@ -11,6 +11,8 @@ Projections without a plan keep the generic method's layout and ``apply``.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import inspect
|
||||
import logging
|
||||
from enum import Enum
|
||||
from typing import Any, ClassVar
|
||||
@@ -64,6 +66,25 @@ def flux3_prims_ts_available() -> bool:
|
||||
return capabilities_satisfied(FLUX3_PRIMS_TS_CAPABILITIES, PlatformInfo.detect())
|
||||
|
||||
|
||||
@functools.cache
|
||||
def flux3_prims_ts_supports_lora() -> bool:
|
||||
"""Whether FlashInfer's PrimsTS GEMMs take the ``delta`` epilogue input."""
|
||||
try:
|
||||
from flashinfer.prims_ts import gemm as prims_ts_gemm
|
||||
except ImportError:
|
||||
return False
|
||||
entry_points = (
|
||||
prims_ts_gemm.fp8_linear,
|
||||
prims_ts_gemm.fp8_linear_swiglu,
|
||||
prims_ts_gemm.fp8_qkv_qknorm_rope,
|
||||
prims_ts_gemm.fp4_linear,
|
||||
prims_ts_gemm.fp4_linear_swiglu,
|
||||
prims_ts_gemm.fp4_qkv_qknorm_rope,
|
||||
prims_ts_gemm.PreparedFp4Linear.__call__,
|
||||
)
|
||||
return all("delta" in inspect.signature(fn).parameters for fn in entry_points)
|
||||
|
||||
|
||||
def _use_separate_qkv_output() -> bool:
|
||||
"""Whether TRT-LLM Gen Sage should receive directly separated Q/K/V."""
|
||||
return envs.SGLANG_USE_SAGE_ATTENTION_IMPL.get().lower() in {
|
||||
@@ -180,7 +201,7 @@ class Flux3PrimsTsMethodMixin:
|
||||
raise NotImplementedError
|
||||
|
||||
def apply_prims_ts_linear(
|
||||
self, layer: nn.Module, x: torch.Tensor, quantized=None
|
||||
self, layer: nn.Module, x: torch.Tensor, quantized=None, *, delta=None
|
||||
) -> torch.Tensor:
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -196,6 +217,7 @@ class Flux3PrimsTsMethodMixin:
|
||||
quantized=None,
|
||||
*,
|
||||
packed_output: bool = False,
|
||||
delta=None,
|
||||
):
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -208,6 +230,8 @@ class Flux3PrimsTsMethodMixin:
|
||||
cos_sin_cache: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
quantized=None,
|
||||
*,
|
||||
delta=None,
|
||||
) -> torch.Tensor:
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -327,12 +351,12 @@ class Flux3Fp8LinearMethod(Flux3PrimsTsMethodMixin, Fp8LinearMethod):
|
||||
return quantize_flux3_activation(x)
|
||||
|
||||
def apply_prims_ts_linear(
|
||||
self, layer: nn.Module, x: torch.Tensor, quantized=None
|
||||
self, layer: nn.Module, x: torch.Tensor, quantized=None, *, delta=None
|
||||
) -> torch.Tensor:
|
||||
from sglang.kernels.ops.diffusion import flux3_clustered_fp8_linear
|
||||
|
||||
return flux3_clustered_fp8_linear(
|
||||
x, layer.weight.t(), layer.weight_scale, quantized
|
||||
x, layer.weight.t(), layer.weight_scale, quantized, delta=delta
|
||||
)
|
||||
|
||||
def apply_prims_ts_swiglu(
|
||||
@@ -342,12 +366,15 @@ class Flux3Fp8LinearMethod(Flux3PrimsTsMethodMixin, Fp8LinearMethod):
|
||||
quantized=None,
|
||||
*,
|
||||
packed_output: bool = False,
|
||||
delta=None,
|
||||
) -> torch.Tensor:
|
||||
from sglang.kernels.ops.diffusion import flux3_fused_mlp_up
|
||||
|
||||
if packed_output:
|
||||
raise NotImplementedError(f"{layer.prefix}: FP8 SwiGLU emits BF16 only")
|
||||
return flux3_fused_mlp_up(x, layer.weight.t(), layer.weight_scale, quantized)
|
||||
return flux3_fused_mlp_up(
|
||||
x, layer.weight.t(), layer.weight_scale, quantized, delta=delta
|
||||
)
|
||||
|
||||
def apply_prims_ts_qkv_qknorm_rope(
|
||||
self,
|
||||
@@ -358,6 +385,8 @@ class Flux3Fp8LinearMethod(Flux3PrimsTsMethodMixin, Fp8LinearMethod):
|
||||
cos_sin_cache: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
quantized=None,
|
||||
*,
|
||||
delta=None,
|
||||
) -> torch.Tensor:
|
||||
from sglang.kernels.ops.diffusion import (
|
||||
flux3_clustered_fp8_qkv_qknorm_rope,
|
||||
@@ -373,6 +402,7 @@ class Flux3Fp8LinearMethod(Flux3PrimsTsMethodMixin, Fp8LinearMethod):
|
||||
positions,
|
||||
quantized,
|
||||
separate_qkv_output=_use_separate_qkv_output(),
|
||||
delta=delta,
|
||||
)
|
||||
|
||||
|
||||
@@ -594,7 +624,7 @@ class Flux3Nvfp4LinearMethod(Flux3PrimsTsMethodMixin, ModelOptFp4LinearMethod):
|
||||
return quantize_flux3_nvfp4_activation(x, layer.prims_ts_input_global_scale)
|
||||
|
||||
def apply_prims_ts_linear(
|
||||
self, layer: nn.Module, x: torch.Tensor, quantized=None
|
||||
self, layer: nn.Module, x: torch.Tensor, quantized=None, *, delta=None
|
||||
) -> torch.Tensor:
|
||||
from sglang.kernels.ops.diffusion import flux3_nvfp4_linear
|
||||
|
||||
@@ -605,6 +635,7 @@ class Flux3Nvfp4LinearMethod(Flux3PrimsTsMethodMixin, ModelOptFp4LinearMethod):
|
||||
layer.prims_ts_input_global_scale,
|
||||
layer.prims_ts_weight_global_scale,
|
||||
quantized,
|
||||
delta=delta,
|
||||
)
|
||||
|
||||
def apply_prims_ts_linear_from_packed(
|
||||
@@ -628,24 +659,34 @@ class Flux3Nvfp4LinearMethod(Flux3PrimsTsMethodMixin, ModelOptFp4LinearMethod):
|
||||
quantized=None,
|
||||
*,
|
||||
packed_output: bool = False,
|
||||
delta=None,
|
||||
):
|
||||
from sglang.kernels.ops.diffusion import (
|
||||
flux3_nvfp4_linear_swiglu,
|
||||
flux3_nvfp4_linear_swiglu_fp4,
|
||||
)
|
||||
|
||||
op = (
|
||||
flux3_nvfp4_linear_swiglu_fp4
|
||||
if packed_output
|
||||
else flux3_nvfp4_linear_swiglu
|
||||
)
|
||||
return op(
|
||||
if packed_output:
|
||||
if delta is not None:
|
||||
raise NotImplementedError(
|
||||
f"{layer.prefix}: packed NVFP4 SwiGLU output has no delta input"
|
||||
)
|
||||
return flux3_nvfp4_linear_swiglu_fp4(
|
||||
x,
|
||||
layer.weight,
|
||||
layer.weight_scale_prims_ts_128x4,
|
||||
layer.prims_ts_input_global_scale,
|
||||
layer.prims_ts_weight_global_scale,
|
||||
quantized,
|
||||
)
|
||||
return flux3_nvfp4_linear_swiglu(
|
||||
x,
|
||||
layer.weight,
|
||||
layer.weight_scale_prims_ts_128x4,
|
||||
layer.prims_ts_input_global_scale,
|
||||
layer.prims_ts_weight_global_scale,
|
||||
quantized,
|
||||
delta=delta,
|
||||
)
|
||||
|
||||
def apply_prims_ts_qkv_qknorm_rope(
|
||||
@@ -657,6 +698,8 @@ class Flux3Nvfp4LinearMethod(Flux3PrimsTsMethodMixin, ModelOptFp4LinearMethod):
|
||||
cos_sin_cache: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
quantized=None,
|
||||
*,
|
||||
delta=None,
|
||||
) -> torch.Tensor:
|
||||
from sglang.kernels.ops.diffusion import (
|
||||
flux3_nvfp4_qkv_qknorm_rope,
|
||||
@@ -675,4 +718,5 @@ class Flux3Nvfp4LinearMethod(Flux3PrimsTsMethodMixin, ModelOptFp4LinearMethod):
|
||||
getattr(layer, "prims_ts_qkv_scale", None),
|
||||
quantized,
|
||||
separate_qkv_output=_use_separate_qkv_output(),
|
||||
delta=delta,
|
||||
)
|
||||
|
||||
@@ -14,6 +14,7 @@ from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
|
||||
from sglang.multimodal_gen.runtime.layers.linear import LinearBase
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.flux3_prims_ts import (
|
||||
Flux3PrimsTsMethodMixin,
|
||||
flux3_prims_ts_supports_lora,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.loader.utils import (
|
||||
get_memory_usage_of_component,
|
||||
@@ -53,6 +54,8 @@ class Flux3ImagePipeline(LoRAPipeline, ComposedPipelineBase):
|
||||
|
||||
def _reject_lora_on_packed_weights(self) -> None:
|
||||
super()._reject_lora_on_packed_weights()
|
||||
if flux3_prims_ts_supports_lora():
|
||||
return
|
||||
for layer in self.get_module("transformer").modules():
|
||||
if (
|
||||
isinstance(layer, LinearBase)
|
||||
@@ -60,7 +63,8 @@ class Flux3ImagePipeline(LoRAPipeline, ComposedPipelineBase):
|
||||
and layer.quant_method.prims_ts_role is not None
|
||||
):
|
||||
raise ValueError(
|
||||
"Dynamic LoRA cannot wrap PrimsTS-prepared FLUX.3 weights. "
|
||||
"Dynamic LoRA cannot wrap PrimsTS-prepared FLUX.3 weights: the "
|
||||
"installed FlashInfer cannot add a LoRA delta to PrimsTS GEMMs. "
|
||||
"Start with --lora-path to load the generic quantized layout."
|
||||
)
|
||||
|
||||
@@ -86,11 +90,13 @@ class Flux3ImagePipeline(LoRAPipeline, ComposedPipelineBase):
|
||||
device = get_local_torch_device()
|
||||
cpu = torch.device("cpu")
|
||||
quant_config = resolve_flux3_image_package_quant_config(package)
|
||||
enable_prims_ts = server_args.lora_path is None
|
||||
enable_prims_ts = (
|
||||
server_args.lora_path is None or flux3_prims_ts_supports_lora()
|
||||
)
|
||||
if not enable_prims_ts and quant_config is not None:
|
||||
logger.info(
|
||||
"FLUX.3 PrimsTS is disabled because dynamic LoRA requires "
|
||||
"the generic quantized linear layout."
|
||||
"FLUX.3 PrimsTS is disabled because the installed FlashInfer "
|
||||
"cannot add a dynamic LoRA delta in its GEMM epilogue."
|
||||
)
|
||||
dit = load_flux3_image_dit(
|
||||
str(package.weights),
|
||||
|
||||
@@ -25,6 +25,7 @@ from sglang.multimodal_gen.runtime.layers.flux3_fused_ops import (
|
||||
flux3_qkv_qknorm_rope,
|
||||
flux3_shared_activation,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.lora.linear import BaseLayerWithLoRA
|
||||
from sglang.multimodal_gen.runtime.layers.quantization import flux3_prims_ts
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import (
|
||||
ConvRotInt8Config,
|
||||
@@ -430,6 +431,15 @@ class _StubLinear:
|
||||
return self.output, None
|
||||
|
||||
|
||||
def _stub_lora(base_layer, delta):
|
||||
"""A LoRA wrapper around a FLUX.3 linear whose adapter adds ``delta``."""
|
||||
stub = mock.NonCallableMock(spec=BaseLayerWithLoRA)
|
||||
stub.base_layer = base_layer
|
||||
stub.lora_active = True
|
||||
stub.fused_lora_delta.return_value = delta
|
||||
return stub
|
||||
|
||||
|
||||
class TestFlux3FusedOps(unittest.TestCase):
|
||||
_CUDA_X = SimpleNamespace(is_cuda=True)
|
||||
|
||||
@@ -531,6 +541,71 @@ class TestFlux3FusedOps(unittest.TestCase):
|
||||
swiglu.assert_called_with(mlp_in, self._CUDA_X, "shared", packed_output=True)
|
||||
from_packed.assert_called_once_with(mlp_out, hidden)
|
||||
|
||||
def test_lora_wrapped_layers_pass_their_delta_to_the_prims_ts_epilogue(self):
|
||||
x = self._CUDA_X
|
||||
linear = mock.Mock(return_value="prims_ts")
|
||||
planned = _StubLinear(Flux3PrimsTsRole.LINEAR, apply_prims_ts_linear=linear)
|
||||
epilogue = mock.Mock(return_value=torch.randn(1, 3, 3 * 2 * 4))
|
||||
qkv = _StubLinear(
|
||||
Flux3PrimsTsRole.QKV_QKNORM_ROPE, apply_prims_ts_qkv_qknorm_rope=epilogue
|
||||
)
|
||||
norm = SimpleNamespace(
|
||||
query_norm=SimpleNamespace(weight="q_w"),
|
||||
key_norm=SimpleNamespace(weight="k_w"),
|
||||
)
|
||||
wrapped = _stub_lora(planned, "delta")
|
||||
with self._platform(10, 0):
|
||||
self.assertEqual(flux3_linear(wrapped, x, "shared"), "prims_ts")
|
||||
flux3_qkv_qknorm_rope(_stub_lora(qkv, "qkv_delta"), x, norm, "c", "p", 2)
|
||||
# A zero runtime LoRA scale adds nothing: the plain kernel runs.
|
||||
flux3_linear(_stub_lora(planned, None), x)
|
||||
wrapped.fused_lora_delta.assert_called_once_with(x, swiglu_rows=False)
|
||||
self.assertEqual(
|
||||
linear.call_args_list,
|
||||
[
|
||||
mock.call(planned, x, "shared", delta="delta"),
|
||||
mock.call(planned, x, None),
|
||||
],
|
||||
)
|
||||
epilogue.assert_called_once_with(
|
||||
qkv, x, "q_w", "k_w", "c", "p", None, delta="qkv_delta"
|
||||
)
|
||||
|
||||
def test_gated_mlp_with_lora_skips_the_packed_handoff(self):
|
||||
"""Either projection's LoRA needs the BF16 SwiGLU output: MLP-up adds its
|
||||
delta in SwiGLU column order, MLP-down computes its delta from it."""
|
||||
x, hidden = self._CUDA_X, SimpleNamespace(is_cuda=True)
|
||||
swiglu = mock.Mock(return_value=hidden)
|
||||
down = mock.Mock(return_value="down")
|
||||
from_packed = mock.Mock(return_value="down_from_packed")
|
||||
mlp_in = _StubLinear(Flux3PrimsTsRole.SWIGLU, apply_prims_ts_swiglu=swiglu)
|
||||
mlp_out = _StubLinear(
|
||||
Flux3PrimsTsRole.LINEAR,
|
||||
apply_prims_ts_linear=down,
|
||||
apply_prims_ts_linear_from_packed=from_packed,
|
||||
)
|
||||
for layer in (mlp_in, mlp_out):
|
||||
layer.quant_method.packed_activation_format = "nvfp4"
|
||||
up_lora = _stub_lora(mlp_in, "up_delta")
|
||||
down_lora = _stub_lora(mlp_out, "down_delta")
|
||||
with self._platform(10, 0):
|
||||
self.assertEqual(flux3_gated_mlp(up_lora, mlp_out, x), "down")
|
||||
self.assertEqual(flux3_gated_mlp(mlp_in, down_lora, x), "down")
|
||||
up_lora.fused_lora_delta.assert_called_once_with(x, swiglu_rows=True)
|
||||
down_lora.fused_lora_delta.assert_called_once_with(hidden, swiglu_rows=False)
|
||||
self.assertEqual(
|
||||
swiglu.call_args_list,
|
||||
[mock.call(mlp_in, x, None, delta="up_delta"), mock.call(mlp_in, x, None)],
|
||||
)
|
||||
self.assertEqual(
|
||||
down.call_args_list,
|
||||
[
|
||||
mock.call(mlp_out, hidden, None),
|
||||
mock.call(mlp_out, hidden, None, delta="down_delta"),
|
||||
],
|
||||
)
|
||||
from_packed.assert_not_called()
|
||||
|
||||
def test_shared_activation_requires_matching_prims_ts_consumers(self):
|
||||
x = self._CUDA_X
|
||||
quantize = mock.Mock(return_value="shared")
|
||||
|
||||
@@ -619,11 +619,16 @@ class TestFlux3RequestControls(unittest.TestCase):
|
||||
|
||||
|
||||
class TestFlux3Lora(unittest.TestCase):
|
||||
def test_rejects_late_lora_before_wrapping_prims_ts_weights(self):
|
||||
_SUPPORTS_LORA = (
|
||||
"sglang.multimodal_gen.runtime.pipelines.flux3_image."
|
||||
"flux3_prims_ts_supports_lora"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _pipeline_with_one_prims_ts_layer():
|
||||
from sglang.multimodal_gen.runtime.layers.linear import LinearBase
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.flux3_prims_ts import (
|
||||
Flux3PrimsTsMethodMixin,
|
||||
Flux3PrimsTsRole,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines.flux3_image import (
|
||||
Flux3ImagePipeline,
|
||||
@@ -635,12 +640,31 @@ class TestFlux3Lora(unittest.TestCase):
|
||||
pipeline = Flux3ImagePipeline.__new__(Flux3ImagePipeline)
|
||||
pipeline.modules = {"transformer": torch.nn.Sequential(layer)}
|
||||
pipeline.lora_target_modules = None
|
||||
pipeline._reject_lora_on_packed_weights()
|
||||
layer.quant_method.prims_ts_role = Flux3PrimsTsRole.SWIGLU
|
||||
with self.assertRaisesRegex(ValueError, "--lora-path"):
|
||||
return pipeline, layer
|
||||
|
||||
def test_rejects_late_lora_before_wrapping_prims_ts_weights(self):
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.flux3_prims_ts import (
|
||||
Flux3PrimsTsRole,
|
||||
)
|
||||
|
||||
pipeline, layer = self._pipeline_with_one_prims_ts_layer()
|
||||
with mock.patch(self._SUPPORTS_LORA, return_value=False):
|
||||
pipeline._reject_lora_on_packed_weights()
|
||||
layer.quant_method.prims_ts_role = Flux3PrimsTsRole.SWIGLU
|
||||
with self.assertRaisesRegex(ValueError, "--lora-path"):
|
||||
pipeline._reject_lora_on_packed_weights()
|
||||
self.assertIs(pipeline.get_module("transformer")[0], layer)
|
||||
|
||||
def test_accepts_late_lora_when_prims_ts_takes_a_delta(self):
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.flux3_prims_ts import (
|
||||
Flux3PrimsTsRole,
|
||||
)
|
||||
|
||||
pipeline, layer = self._pipeline_with_one_prims_ts_layer()
|
||||
layer.quant_method.prims_ts_role = Flux3PrimsTsRole.SWIGLU
|
||||
with mock.patch(self._SUPPORTS_LORA, return_value=True):
|
||||
pipeline._reject_lora_on_packed_weights()
|
||||
|
||||
def test_prefixed_adapter_keys_reach_the_native_modules(self):
|
||||
"""Prefixed adapters (BFL/ostris ``dit.``, diffusers ``transformer.``)
|
||||
must map onto the native DiT modules, not load onto 0 layers, and load
|
||||
|
||||
@@ -5,7 +5,8 @@ request scale, add) and only drops multiplications by 1.0, which are exact. At
|
||||
"lossless" the second GEMM accumulates the scaled delta onto the base output and
|
||||
rounds once, and a row-parallel bias moves into the base GEMM's epilogue. A
|
||||
forward whose stage does not put its request in the forward context runs the
|
||||
exact chain.
|
||||
exact chain. ``fused_lora_delta`` returns the same delta for GEMM epilogues
|
||||
that add it themselves.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
@@ -17,7 +18,10 @@ import torch.nn.functional as F
|
||||
from torch import nn
|
||||
|
||||
from sglang.multimodal_gen.runtime.layers.lora.linear import (
|
||||
BaseLayerWithLoRA,
|
||||
ColumnParallelLinearWithLoRA,
|
||||
LinearWithLoRA,
|
||||
MergedColumnParallelLinearWithLoRA,
|
||||
RowParallelLinearWithLoRA,
|
||||
_compute_lora_delta,
|
||||
_request_allows_lossless,
|
||||
@@ -188,3 +192,109 @@ def test_row_parallel_bias_moves_into_the_gemm_only_above_exact():
|
||||
candidate=lossless,
|
||||
label="row-parallel LoRA with the bias in the GEMM",
|
||||
)
|
||||
|
||||
|
||||
class _ColumnLinear(nn.Module):
|
||||
"""The ColumnParallelLinear surface the LoRA wrapper reads, at TP 1."""
|
||||
|
||||
def __init__(self, output_sizes: list[int]):
|
||||
super().__init__()
|
||||
g = torch.Generator().manual_seed(6)
|
||||
self.weight = nn.Parameter(
|
||||
torch.randn(sum(output_sizes), IN_DIM, generator=g) * 0.05
|
||||
)
|
||||
self.bias = None
|
||||
self.output_sizes = output_sizes
|
||||
self.output_partition_sizes = output_sizes
|
||||
self.skip_bias_add = False
|
||||
self.gather_output = False
|
||||
self.quant_method = SimpleNamespace(
|
||||
apply=lambda layer, x, bias=None: F.linear(x, layer.weight, bias)
|
||||
)
|
||||
|
||||
def forward(self, x):
|
||||
return F.linear(x, self.weight), None
|
||||
|
||||
|
||||
def _column_layer(*, stacked=False, alpha=RANK, strength=1.0, output_offset=None):
|
||||
a, b = _adapter(stacked, torch.float32)
|
||||
if stacked:
|
||||
layer = MergedColumnParallelLinearWithLoRA(
|
||||
_ColumnLinear([OUT_DIM // 3] * 3), lora_rank=RANK, lora_alpha=alpha
|
||||
)
|
||||
else:
|
||||
layer = ColumnParallelLinearWithLoRA(
|
||||
_ColumnLinear([OUT_DIM]), lora_rank=RANK, lora_alpha=alpha
|
||||
)
|
||||
layer.set_lora_weights(
|
||||
a, b, strength=strength, merge_weights=False, output_offset=output_offset
|
||||
)
|
||||
return layer
|
||||
|
||||
|
||||
def _forward_delta(layer, x):
|
||||
with patch(_TP_RANK_PATCH, return_value=0):
|
||||
out, _ = layer(x)
|
||||
base, _ = layer.base_layer(x)
|
||||
return (out - base).reshape(-1, out.shape[-1])
|
||||
|
||||
|
||||
def _fused_delta(layer, x, **kwargs):
|
||||
with patch(_TP_RANK_PATCH, return_value=0):
|
||||
return layer.fused_lora_delta(x, **kwargs)
|
||||
|
||||
|
||||
def _assert_fused_delta(actual, expected):
|
||||
assert actual.dtype == torch.bfloat16 and actual.is_contiguous()
|
||||
torch.testing.assert_close(actual.float(), expected, rtol=2e-2, atol=2e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stacked", [False, True])
|
||||
def test_fused_delta_matches_the_forward_delta(stacked):
|
||||
layer = _column_layer(stacked=stacked, alpha=2 * RANK, strength=0.7)
|
||||
x = _activations(TOKENS, IN_DIM, seed=7).reshape(2, TOKENS // 2, IN_DIM)
|
||||
delta = _fused_delta(layer, x)
|
||||
assert delta.shape == (TOKENS, OUT_DIM)
|
||||
_assert_fused_delta(delta, _forward_delta(layer, x))
|
||||
|
||||
|
||||
def test_fused_delta_follows_the_runtime_scale_and_output_offset():
|
||||
layer = _column_layer(output_offset=torch.randn(OUT_DIM))
|
||||
x = _activations(TOKENS, IN_DIM, seed=8)
|
||||
with patch.object(BaseLayerWithLoRA, "_runtime_lora_scale", return_value=0.5):
|
||||
_assert_fused_delta(_fused_delta(layer, x), _forward_delta(layer, x))
|
||||
with patch.object(BaseLayerWithLoRA, "_runtime_lora_scale", return_value=0.0):
|
||||
assert _fused_delta(layer, x) is None
|
||||
|
||||
|
||||
def test_fused_delta_interleaves_swiglu_value_and_gate_columns():
|
||||
layer = _column_layer(output_offset=torch.randn(OUT_DIM))
|
||||
x = _activations(TOKENS, IN_DIM, seed=9)
|
||||
gate, value = _forward_delta(layer, x).chunk(2, dim=-1)
|
||||
interleaved = torch.stack((value, gate), dim=-1).flatten(-2)
|
||||
_assert_fused_delta(_fused_delta(layer, x, swiglu_rows=True), interleaved)
|
||||
|
||||
|
||||
def test_fused_delta_drops_cached_factors_for_new_adapter_weights():
|
||||
layer = _column_layer()
|
||||
x = _activations(TOKENS, IN_DIM, seed=10)
|
||||
_fused_delta(layer, x)
|
||||
a, b = _adapter(False, torch.float32, seed=11)
|
||||
layer.set_lora_weights(a, b, merge_weights=False)
|
||||
_assert_fused_delta(_fused_delta(layer, x), _forward_delta(layer, x))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("state", ["no_adapter", "disabled", "merged"])
|
||||
def test_inactive_layers_have_no_fused_delta(state):
|
||||
if state == "no_adapter":
|
||||
layer = ColumnParallelLinearWithLoRA(
|
||||
_ColumnLinear([OUT_DIM]), lora_rank=RANK, lora_alpha=RANK
|
||||
)
|
||||
else:
|
||||
layer = _column_layer()
|
||||
if state == "disabled":
|
||||
layer.disable_lora = True
|
||||
if state == "merged":
|
||||
layer.merged = True
|
||||
assert not layer.lora_active
|
||||
assert _fused_delta(layer, _activations(TOKENS, IN_DIM, seed=12)) is None
|
||||
|
||||
Reference in New Issue
Block a user