[Perf] [NVIDIA] diffusion/FLUX.3: Run dynamic LoRA through PrimsTS GEMMs (#44761)

Co-authored-by: Mick <mickjagger19@icloud.com>
This commit is contained in:
Linda
2026-10-18 20:32:19 +08:00
committed by GitHub
co-authored by Mick
parent 1fd46ddd56
commit 680a2ff27e
10 changed files with 432 additions and 33 deletions
@@ -9,6 +9,7 @@ import torch
from flashinfer.prims_ts import gemm as prims_ts_gemm
from sglang.kernels.ops.diffusion.gemm.flux3_prims_ts_config import (
delta_kwargs,
flux3_prims_ts_config,
)
from sglang.kernels.ops.quantization.per_token_quant_fp8 import per_token_quant_fp8
@@ -90,6 +91,8 @@ def flux3_fused_mlp_up(
packed_weight: torch.Tensor,
packed_weight_scale: torch.Tensor,
quantized: QuantizedActivation | None = None,
*,
delta: torch.Tensor | None = None,
) -> torch.Tensor:
"""Run fused FP8 GEMM + SwiGLU, quantizing BF16 rows when needed."""
if x.dtype != torch.bfloat16:
@@ -103,6 +106,7 @@ def flux3_fused_mlp_up(
config=_flux3_gemm_config(
x, epilogue="swiglu", k=packed_weight.shape[1], n=packed_weight.shape[0]
),
**delta_kwargs(delta),
skip_check=True,
)
return out.view(*x.shape[:-1], out.shape[-1])
@@ -114,6 +118,8 @@ def flux3_clustered_fp8_linear(
packed_weight_scale: torch.Tensor,
quantized: QuantizedActivation | None = None,
qkv_scale: torch.Tensor | None = None,
*,
delta: torch.Tensor | None = None,
) -> torch.Tensor:
"""Run a PrimsTS FP8 GEMM, optionally scaling packed Q/K/V thirds."""
if x.dtype != torch.bfloat16:
@@ -132,6 +138,7 @@ def flux3_clustered_fp8_linear(
n=packed_weight.shape[0],
qkv_scale=qkv_scale is not None,
),
**delta_kwargs(delta),
skip_check=True,
)
return out.view(*x.shape[:-1], out.shape[-1])
@@ -149,6 +156,7 @@ def flux3_clustered_fp8_qkv_qknorm_rope(
qkv_scale: torch.Tensor | None = None,
*,
separate_qkv_output: bool = False,
delta: torch.Tensor | None = None,
) -> torch.Tensor:
"""Fuse QKV GEMM, QK RMSNorm, and RoPE, quantizing rows when needed."""
if x.dtype != torch.bfloat16:
@@ -194,6 +202,7 @@ def flux3_clustered_fp8_qkv_qknorm_rope(
head_dim=q_norm_weight.numel(),
qkv_scale=qkv_scale,
config=config,
**delta_kwargs(delta),
skip_check=True,
)
if separate_qkv_output:
@@ -16,6 +16,7 @@ from flashinfer import SfLayout, nvfp4_quantize
from flashinfer.prims_ts import gemm as prims_ts_gemm
from sglang.kernels.ops.diffusion.gemm.flux3_prims_ts_config import (
delta_kwargs,
flux3_prims_ts_config,
)
@@ -253,6 +254,8 @@ def flux3_nvfp4_linear(
input_global_scale: float,
weight_global_scale: float,
quantized: QuantizedNvfp4Activation | None = None,
*,
delta: torch.Tensor | None = None,
) -> torch.Tensor:
"""Run a FLUX.3 projection entirely through PrimsTS NVFP4 GEMM."""
a_packed, a_scale_128x4, _ = _quantized_activation(x, input_global_scale, quantized)
@@ -270,7 +273,7 @@ def flux3_nvfp4_linear(
),
)
if prepared is not None:
out = prepared(a_packed, a_scale_128x4)
out = prepared(a_packed, a_scale_128x4, **delta_kwargs(delta))
else:
out = prims_ts_gemm.fp4_linear(
a_packed,
@@ -279,6 +282,7 @@ def flux3_nvfp4_linear(
weight_packed,
weight_scale_128x4,
weight_global_scale,
**delta_kwargs(delta),
skip_check=True,
)
return out.view(*x.shape[:-1], out.shape[-1])
@@ -328,6 +332,8 @@ def flux3_nvfp4_linear_swiglu(
input_global_scale: float,
weight_global_scale: float,
quantized: QuantizedNvfp4Activation | None = None,
*,
delta: torch.Tensor | None = None,
) -> torch.Tensor:
"""Run FLUX.3's gated MLP-up projection with PrimsTS' SwiGLU epilogue."""
a_packed, a_scale_128x4, _ = _quantized_activation(x, input_global_scale, quantized)
@@ -347,7 +353,7 @@ def flux3_nvfp4_linear_swiglu(
device=x.device,
)
if prepared is not None:
out = prepared(a_packed, a_scale_128x4)
out = prepared(a_packed, a_scale_128x4, **delta_kwargs(delta))
else:
out = prims_ts_gemm.fp4_linear_swiglu(
a_packed,
@@ -356,6 +362,7 @@ def flux3_nvfp4_linear_swiglu(
weight_packed,
weight_scale_128x4,
weight_global_scale,
**delta_kwargs(delta),
skip_check=True,
)
return out.view(*x.shape[:-1], out.shape[-1])
@@ -425,6 +432,7 @@ def flux3_nvfp4_qkv_qknorm_rope(
quantized: QuantizedNvfp4Activation | None = None,
*,
separate_qkv_output: bool = False,
delta: torch.Tensor | None = None,
) -> torch.Tensor:
"""Fuse NVFP4 QKV GEMM, Q/K RMSNorm, and RoPE through PrimsTS.
@@ -458,6 +466,7 @@ def flux3_nvfp4_qkv_qknorm_rope(
a_scale_128x4,
cos_sin=cos_sin_cache,
positions=positions,
**delta_kwargs(delta),
)
elif separate_qkv_output:
raise RuntimeError("separate QKV output requires a prepared CUDA projection")
@@ -479,6 +488,7 @@ def flux3_nvfp4_qkv_qknorm_rope(
head_dim=head_dim,
is_neox=False,
qkv_scale=qkv_scale,
**delta_kwargs(delta),
skip_check=True,
)
if separate_qkv_output:
@@ -94,6 +94,11 @@ _TACTICS = {
}
def delta_kwargs(delta: torch.Tensor | None) -> dict[str, torch.Tensor]:
# FlashInfer releases without the PrimsTS delta input reject the keyword.
return {} if delta is None else {"delta": delta}
@functools.cache
def _tmem_overlap_epilogues() -> frozenset[str]:
# FlashInfer PR 6255 (SwiGLU) and 6286 (QKV) add tmem_overlap to these prepare
@@ -8,6 +8,8 @@ plus SGLang's fused QK-norm/RoPE and SwiGLU kernels, on any device and format.
FLUX.3 quant method (:mod:`...quantization.flux3_prims_ts`) prepared at load
time. FlashInfer is selected per call, only on SM10x and only for layers whose
method planned the matching role, so call sites never branch on the backend.
A LoRA-wrapped layer runs its base layer's PrimsTS epilogue with the adapter's
delta added before the activation, QK-norm, or output conversion.
"""
from __future__ import annotations
@@ -18,15 +20,35 @@ from torch import nn
from sglang.kernels.fused_op import BaseFusedOp, register_fused_op
from sglang.kernels.spec import FormatSignature, KernelBackend
from sglang.multimodal_gen.runtime.layers.layernorm import apply_qk_norm_rope
from sglang.multimodal_gen.runtime.layers.lora.linear import BaseLayerWithLoRA
from sglang.multimodal_gen.runtime.layers.quantization.flux3_prims_ts import (
FLUX3_PRIMS_TS_CAPABILITIES,
Flux3PrimsTsRole,
)
def _base_layer(layer: nn.Module) -> nn.Module:
"""The quantized layer under an optional LoRA wrapper."""
return layer.base_layer if isinstance(layer, BaseLayerWithLoRA) else layer
def _lora_active(layer: nn.Module) -> bool:
return isinstance(layer, BaseLayerWithLoRA) and layer.lora_active
def _lora_delta_kwargs(
layer: nn.Module, x: torch.Tensor, *, swiglu_rows: bool = False
) -> dict[str, torch.Tensor]:
"""The ``delta`` kwarg while the LoRA adds something; old FlashInfer rejects it."""
if not _lora_active(layer):
return {}
delta = layer.fused_lora_delta(x, swiglu_rows=swiglu_rows)
return {} if delta is None else {"delta": delta}
def _prims_ts_method(layer: nn.Module, role: Flux3PrimsTsRole):
"""The layer's quant method if it planned ``role`` for PrimsTS, else None."""
method = getattr(layer, "quant_method", None)
method = getattr(_base_layer(layer), "quant_method", None)
if getattr(method, "prims_ts_role", None) is role:
return method
return None
@@ -93,7 +115,10 @@ class Flux3LinearOp(_Flux3PrimsTsOp):
return out
def forward_flashinfer(self, layer, x, quantized=None) -> torch.Tensor:
return layer.quant_method.apply_prims_ts_linear(layer, x, quantized)
base = _base_layer(layer)
return base.quant_method.apply_prims_ts_linear(
base, x, quantized, **_lora_delta_kwargs(layer, x)
)
class Flux3QKVQKNormRopeOp(_Flux3PrimsTsOp):
@@ -154,14 +179,16 @@ class Flux3QKVQKNormRopeOp(_Flux3PrimsTsOp):
local_heads,
quantized=None,
):
qkv = layer.quant_method.apply_prims_ts_qkv_qknorm_rope(
layer,
base = _base_layer(layer)
qkv = base.quant_method.apply_prims_ts_qkv_qknorm_rope(
base,
x,
norm.query_norm.weight,
norm.key_norm.weight,
cos_sin_cache,
positions,
quantized,
**_lora_delta_kwargs(layer, x),
)
return _unpack_qkv(qkv, local_heads)
@@ -185,17 +212,25 @@ class Flux3GatedMLPOp(_Flux3PrimsTsOp):
return flux3_linear(mlp_out, _silu_and_mul(mlp))
def forward_flashinfer(self, mlp_in, mlp_out, x, quantized=None) -> torch.Tensor:
up = mlp_in.quant_method
up_layer = _base_layer(mlp_in)
up = up_layer.quant_method
down = _prims_ts_method(mlp_out, Flux3PrimsTsRole.LINEAR)
up_delta = _lora_delta_kwargs(mlp_in, x, swiglu_rows=True)
if (
down is not None
and not up_delta
and not _lora_active(mlp_out)
and up.packed_activation_format is not None
and down.packed_activation_format == up.packed_activation_format
):
# Hand MLP-down the packed SwiGLU output without a BF16 round trip.
packed = up.apply_prims_ts_swiglu(mlp_in, x, quantized, packed_output=True)
return down.apply_prims_ts_linear_from_packed(mlp_out, packed)
return flux3_linear(mlp_out, up.apply_prims_ts_swiglu(mlp_in, x, quantized))
packed = up.apply_prims_ts_swiglu(
up_layer, x, quantized, packed_output=True
)
return down.apply_prims_ts_linear_from_packed(_base_layer(mlp_out), packed)
# A LoRA on MLP-down needs the BF16 SwiGLU output for its own delta.
hidden = up.apply_prims_ts_swiglu(up_layer, x, quantized, **up_delta)
return flux3_linear(mlp_out, hidden)
_FLUX3_LINEAR = register_fused_op(Flux3LinearOp(), __name__, "_FLUX3_LINEAR")
@@ -241,11 +276,11 @@ def flux3_shared_activation(x: torch.Tensor, qkv: nn.Module, mlp_in: nn.Module):
if not x.is_cuda:
return None
up = _prims_ts_method(mlp_in, Flux3PrimsTsRole.SWIGLU)
qkv_method = getattr(qkv, "quant_method", None)
qkv_method = getattr(_base_layer(qkv), "quant_method", None)
if (
up is None
or getattr(qkv_method, "prims_ts_role", None) is None
or qkv_method.activation_format != up.activation_format
):
return None
return up.quantize_prims_ts_activation(mlp_in, x)
return up.quantize_prims_ts_activation(_base_layer(mlp_in), x)
@@ -180,6 +180,7 @@ class BaseLayerWithLoRA(nn.Module):
self.lora_B = None
self.lora_output_offset = None
self.has_lora_output_offset = False
self._fused_lora_cache: tuple[tuple, torch.Tensor, torch.Tensor] | None = None
@property
def weight(self):
@@ -276,6 +277,86 @@ class BaseLayerWithLoRA(nn.Module):
def slice_lora_b_weights(self, B: torch.Tensor) -> torch.Tensor:
return B
@property
def lora_active(self) -> bool:
"""Whether ``forward`` adds anything to the base layer's output."""
if self.disable_lora:
return False
if self.merged:
return self.has_lora_output_offset
return self.lora_A is not None
@staticmethod
def _interleave_swiglu_rows(t: torch.Tensor) -> torch.Tensor:
# [gate | value] rows -> [value_0, gate_0, value_1, gate_1, ...]
half = t.shape[0] // 2
return torch.stack((t[half:], t[:half]), dim=1).reshape(t.shape)
def _fused_lora_factors(
self, device: torch.device, swiglu_rows: bool
) -> tuple[torch.Tensor, torch.Tensor]:
lora_A, lora_B = self.lora_A, self.lora_B
if isinstance(lora_B, DTensor):
lora_A, lora_B = lora_A.to_local(), lora_B.to_local()
key = (
lora_A.data_ptr(),
lora_A._version,
lora_B.data_ptr(),
lora_B._version,
swiglu_rows,
device,
)
cached = self._fused_lora_cache
if cached is not None and cached[0] == key:
return cached[1], cached[2]
A = self.slice_lora_a_weights(lora_A.to(device))
B = self.slice_lora_b_weights(lora_B.to(device))
if A.dim() == 3:
# Stacked groups: one shrink GEMM, then a block-diagonal expand.
A = A.reshape(-1, A.shape[-1])
B = torch.block_diag(*B.unbind(0))
if swiglu_rows:
B = self._interleave_swiglu_rows(B)
A, B = A.contiguous(), B.contiguous()
self._fused_lora_cache = (key, A, B)
return A, B
def fused_lora_delta(
self, x: torch.Tensor, *, swiglu_rows: bool = False
) -> torch.Tensor | None:
"""``forward(x) - base_layer(x)`` as a contiguous BF16 ``[tokens, out]``.
For GEMM epilogues that add the adapter's update themselves. None when
there is nothing to add. ``swiglu_rows`` orders the columns like the
interleaved FLUX.3 PrimsTS SwiGLU weight (``[value_i, gate_i]`` pairs).
"""
if not self.lora_active:
return None
runtime_lora_scale = 1.0 if self.merged else self._runtime_lora_scale()
if runtime_lora_scale == 0.0:
return None
x_2d = x.reshape(-1, x.shape[-1])
delta = None
if not self.merged:
A, B = self._fused_lora_factors(device=x.device, swiglu_rows=swiglu_rows)
scale = self.strength * runtime_lora_scale
if self.lora_alpha != self.lora_rank:
scale *= self.lora_alpha / self.lora_rank # type: ignore
hidden = x_2d.to(dtype=A.dtype) @ A.T
if scale != 1.0:
hidden = hidden * scale
delta = hidden @ B.T
offset = self._active_lora_output_offset()
if offset is not None:
offset = offset.to(device=x.device) * runtime_lora_scale
if swiglu_rows:
offset = self._interleave_swiglu_rows(offset)
if delta is None:
delta = offset.expand(x_2d.shape[0], -1)
else:
delta = delta + offset.to(dtype=delta.dtype)
return delta.to(torch.bfloat16).contiguous()
def _scaled_lora_output_offset(
self,
offset: torch.Tensor | None,
@@ -11,6 +11,8 @@ Projections without a plan keep the generic method's layout and ``apply``.
from __future__ import annotations
import functools
import inspect
import logging
from enum import Enum
from typing import Any, ClassVar
@@ -64,6 +66,25 @@ def flux3_prims_ts_available() -> bool:
return capabilities_satisfied(FLUX3_PRIMS_TS_CAPABILITIES, PlatformInfo.detect())
@functools.cache
def flux3_prims_ts_supports_lora() -> bool:
"""Whether FlashInfer's PrimsTS GEMMs take the ``delta`` epilogue input."""
try:
from flashinfer.prims_ts import gemm as prims_ts_gemm
except ImportError:
return False
entry_points = (
prims_ts_gemm.fp8_linear,
prims_ts_gemm.fp8_linear_swiglu,
prims_ts_gemm.fp8_qkv_qknorm_rope,
prims_ts_gemm.fp4_linear,
prims_ts_gemm.fp4_linear_swiglu,
prims_ts_gemm.fp4_qkv_qknorm_rope,
prims_ts_gemm.PreparedFp4Linear.__call__,
)
return all("delta" in inspect.signature(fn).parameters for fn in entry_points)
def _use_separate_qkv_output() -> bool:
"""Whether TRT-LLM Gen Sage should receive directly separated Q/K/V."""
return envs.SGLANG_USE_SAGE_ATTENTION_IMPL.get().lower() in {
@@ -180,7 +201,7 @@ class Flux3PrimsTsMethodMixin:
raise NotImplementedError
def apply_prims_ts_linear(
self, layer: nn.Module, x: torch.Tensor, quantized=None
self, layer: nn.Module, x: torch.Tensor, quantized=None, *, delta=None
) -> torch.Tensor:
raise NotImplementedError
@@ -196,6 +217,7 @@ class Flux3PrimsTsMethodMixin:
quantized=None,
*,
packed_output: bool = False,
delta=None,
):
raise NotImplementedError
@@ -208,6 +230,8 @@ class Flux3PrimsTsMethodMixin:
cos_sin_cache: torch.Tensor,
positions: torch.Tensor,
quantized=None,
*,
delta=None,
) -> torch.Tensor:
raise NotImplementedError
@@ -327,12 +351,12 @@ class Flux3Fp8LinearMethod(Flux3PrimsTsMethodMixin, Fp8LinearMethod):
return quantize_flux3_activation(x)
def apply_prims_ts_linear(
self, layer: nn.Module, x: torch.Tensor, quantized=None
self, layer: nn.Module, x: torch.Tensor, quantized=None, *, delta=None
) -> torch.Tensor:
from sglang.kernels.ops.diffusion import flux3_clustered_fp8_linear
return flux3_clustered_fp8_linear(
x, layer.weight.t(), layer.weight_scale, quantized
x, layer.weight.t(), layer.weight_scale, quantized, delta=delta
)
def apply_prims_ts_swiglu(
@@ -342,12 +366,15 @@ class Flux3Fp8LinearMethod(Flux3PrimsTsMethodMixin, Fp8LinearMethod):
quantized=None,
*,
packed_output: bool = False,
delta=None,
) -> torch.Tensor:
from sglang.kernels.ops.diffusion import flux3_fused_mlp_up
if packed_output:
raise NotImplementedError(f"{layer.prefix}: FP8 SwiGLU emits BF16 only")
return flux3_fused_mlp_up(x, layer.weight.t(), layer.weight_scale, quantized)
return flux3_fused_mlp_up(
x, layer.weight.t(), layer.weight_scale, quantized, delta=delta
)
def apply_prims_ts_qkv_qknorm_rope(
self,
@@ -358,6 +385,8 @@ class Flux3Fp8LinearMethod(Flux3PrimsTsMethodMixin, Fp8LinearMethod):
cos_sin_cache: torch.Tensor,
positions: torch.Tensor,
quantized=None,
*,
delta=None,
) -> torch.Tensor:
from sglang.kernels.ops.diffusion import (
flux3_clustered_fp8_qkv_qknorm_rope,
@@ -373,6 +402,7 @@ class Flux3Fp8LinearMethod(Flux3PrimsTsMethodMixin, Fp8LinearMethod):
positions,
quantized,
separate_qkv_output=_use_separate_qkv_output(),
delta=delta,
)
@@ -594,7 +624,7 @@ class Flux3Nvfp4LinearMethod(Flux3PrimsTsMethodMixin, ModelOptFp4LinearMethod):
return quantize_flux3_nvfp4_activation(x, layer.prims_ts_input_global_scale)
def apply_prims_ts_linear(
self, layer: nn.Module, x: torch.Tensor, quantized=None
self, layer: nn.Module, x: torch.Tensor, quantized=None, *, delta=None
) -> torch.Tensor:
from sglang.kernels.ops.diffusion import flux3_nvfp4_linear
@@ -605,6 +635,7 @@ class Flux3Nvfp4LinearMethod(Flux3PrimsTsMethodMixin, ModelOptFp4LinearMethod):
layer.prims_ts_input_global_scale,
layer.prims_ts_weight_global_scale,
quantized,
delta=delta,
)
def apply_prims_ts_linear_from_packed(
@@ -628,24 +659,34 @@ class Flux3Nvfp4LinearMethod(Flux3PrimsTsMethodMixin, ModelOptFp4LinearMethod):
quantized=None,
*,
packed_output: bool = False,
delta=None,
):
from sglang.kernels.ops.diffusion import (
flux3_nvfp4_linear_swiglu,
flux3_nvfp4_linear_swiglu_fp4,
)
op = (
flux3_nvfp4_linear_swiglu_fp4
if packed_output
else flux3_nvfp4_linear_swiglu
)
return op(
if packed_output:
if delta is not None:
raise NotImplementedError(
f"{layer.prefix}: packed NVFP4 SwiGLU output has no delta input"
)
return flux3_nvfp4_linear_swiglu_fp4(
x,
layer.weight,
layer.weight_scale_prims_ts_128x4,
layer.prims_ts_input_global_scale,
layer.prims_ts_weight_global_scale,
quantized,
)
return flux3_nvfp4_linear_swiglu(
x,
layer.weight,
layer.weight_scale_prims_ts_128x4,
layer.prims_ts_input_global_scale,
layer.prims_ts_weight_global_scale,
quantized,
delta=delta,
)
def apply_prims_ts_qkv_qknorm_rope(
@@ -657,6 +698,8 @@ class Flux3Nvfp4LinearMethod(Flux3PrimsTsMethodMixin, ModelOptFp4LinearMethod):
cos_sin_cache: torch.Tensor,
positions: torch.Tensor,
quantized=None,
*,
delta=None,
) -> torch.Tensor:
from sglang.kernels.ops.diffusion import (
flux3_nvfp4_qkv_qknorm_rope,
@@ -675,4 +718,5 @@ class Flux3Nvfp4LinearMethod(Flux3PrimsTsMethodMixin, ModelOptFp4LinearMethod):
getattr(layer, "prims_ts_qkv_scale", None),
quantized,
separate_qkv_output=_use_separate_qkv_output(),
delta=delta,
)
@@ -14,6 +14,7 @@ from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
from sglang.multimodal_gen.runtime.layers.linear import LinearBase
from sglang.multimodal_gen.runtime.layers.quantization.flux3_prims_ts import (
Flux3PrimsTsMethodMixin,
flux3_prims_ts_supports_lora,
)
from sglang.multimodal_gen.runtime.loader.utils import (
get_memory_usage_of_component,
@@ -53,6 +54,8 @@ class Flux3ImagePipeline(LoRAPipeline, ComposedPipelineBase):
def _reject_lora_on_packed_weights(self) -> None:
super()._reject_lora_on_packed_weights()
if flux3_prims_ts_supports_lora():
return
for layer in self.get_module("transformer").modules():
if (
isinstance(layer, LinearBase)
@@ -60,7 +63,8 @@ class Flux3ImagePipeline(LoRAPipeline, ComposedPipelineBase):
and layer.quant_method.prims_ts_role is not None
):
raise ValueError(
"Dynamic LoRA cannot wrap PrimsTS-prepared FLUX.3 weights. "
"Dynamic LoRA cannot wrap PrimsTS-prepared FLUX.3 weights: the "
"installed FlashInfer cannot add a LoRA delta to PrimsTS GEMMs. "
"Start with --lora-path to load the generic quantized layout."
)
@@ -86,11 +90,13 @@ class Flux3ImagePipeline(LoRAPipeline, ComposedPipelineBase):
device = get_local_torch_device()
cpu = torch.device("cpu")
quant_config = resolve_flux3_image_package_quant_config(package)
enable_prims_ts = server_args.lora_path is None
enable_prims_ts = (
server_args.lora_path is None or flux3_prims_ts_supports_lora()
)
if not enable_prims_ts and quant_config is not None:
logger.info(
"FLUX.3 PrimsTS is disabled because dynamic LoRA requires "
"the generic quantized linear layout."
"FLUX.3 PrimsTS is disabled because the installed FlashInfer "
"cannot add a dynamic LoRA delta in its GEMM epilogue."
)
dit = load_flux3_image_dit(
str(package.weights),
@@ -25,6 +25,7 @@ from sglang.multimodal_gen.runtime.layers.flux3_fused_ops import (
flux3_qkv_qknorm_rope,
flux3_shared_activation,
)
from sglang.multimodal_gen.runtime.layers.lora.linear import BaseLayerWithLoRA
from sglang.multimodal_gen.runtime.layers.quantization import flux3_prims_ts
from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import (
ConvRotInt8Config,
@@ -430,6 +431,15 @@ class _StubLinear:
return self.output, None
def _stub_lora(base_layer, delta):
"""A LoRA wrapper around a FLUX.3 linear whose adapter adds ``delta``."""
stub = mock.NonCallableMock(spec=BaseLayerWithLoRA)
stub.base_layer = base_layer
stub.lora_active = True
stub.fused_lora_delta.return_value = delta
return stub
class TestFlux3FusedOps(unittest.TestCase):
_CUDA_X = SimpleNamespace(is_cuda=True)
@@ -531,6 +541,71 @@ class TestFlux3FusedOps(unittest.TestCase):
swiglu.assert_called_with(mlp_in, self._CUDA_X, "shared", packed_output=True)
from_packed.assert_called_once_with(mlp_out, hidden)
def test_lora_wrapped_layers_pass_their_delta_to_the_prims_ts_epilogue(self):
x = self._CUDA_X
linear = mock.Mock(return_value="prims_ts")
planned = _StubLinear(Flux3PrimsTsRole.LINEAR, apply_prims_ts_linear=linear)
epilogue = mock.Mock(return_value=torch.randn(1, 3, 3 * 2 * 4))
qkv = _StubLinear(
Flux3PrimsTsRole.QKV_QKNORM_ROPE, apply_prims_ts_qkv_qknorm_rope=epilogue
)
norm = SimpleNamespace(
query_norm=SimpleNamespace(weight="q_w"),
key_norm=SimpleNamespace(weight="k_w"),
)
wrapped = _stub_lora(planned, "delta")
with self._platform(10, 0):
self.assertEqual(flux3_linear(wrapped, x, "shared"), "prims_ts")
flux3_qkv_qknorm_rope(_stub_lora(qkv, "qkv_delta"), x, norm, "c", "p", 2)
# A zero runtime LoRA scale adds nothing: the plain kernel runs.
flux3_linear(_stub_lora(planned, None), x)
wrapped.fused_lora_delta.assert_called_once_with(x, swiglu_rows=False)
self.assertEqual(
linear.call_args_list,
[
mock.call(planned, x, "shared", delta="delta"),
mock.call(planned, x, None),
],
)
epilogue.assert_called_once_with(
qkv, x, "q_w", "k_w", "c", "p", None, delta="qkv_delta"
)
def test_gated_mlp_with_lora_skips_the_packed_handoff(self):
"""Either projection's LoRA needs the BF16 SwiGLU output: MLP-up adds its
delta in SwiGLU column order, MLP-down computes its delta from it."""
x, hidden = self._CUDA_X, SimpleNamespace(is_cuda=True)
swiglu = mock.Mock(return_value=hidden)
down = mock.Mock(return_value="down")
from_packed = mock.Mock(return_value="down_from_packed")
mlp_in = _StubLinear(Flux3PrimsTsRole.SWIGLU, apply_prims_ts_swiglu=swiglu)
mlp_out = _StubLinear(
Flux3PrimsTsRole.LINEAR,
apply_prims_ts_linear=down,
apply_prims_ts_linear_from_packed=from_packed,
)
for layer in (mlp_in, mlp_out):
layer.quant_method.packed_activation_format = "nvfp4"
up_lora = _stub_lora(mlp_in, "up_delta")
down_lora = _stub_lora(mlp_out, "down_delta")
with self._platform(10, 0):
self.assertEqual(flux3_gated_mlp(up_lora, mlp_out, x), "down")
self.assertEqual(flux3_gated_mlp(mlp_in, down_lora, x), "down")
up_lora.fused_lora_delta.assert_called_once_with(x, swiglu_rows=True)
down_lora.fused_lora_delta.assert_called_once_with(hidden, swiglu_rows=False)
self.assertEqual(
swiglu.call_args_list,
[mock.call(mlp_in, x, None, delta="up_delta"), mock.call(mlp_in, x, None)],
)
self.assertEqual(
down.call_args_list,
[
mock.call(mlp_out, hidden, None),
mock.call(mlp_out, hidden, None, delta="down_delta"),
],
)
from_packed.assert_not_called()
def test_shared_activation_requires_matching_prims_ts_consumers(self):
x = self._CUDA_X
quantize = mock.Mock(return_value="shared")
@@ -619,11 +619,16 @@ class TestFlux3RequestControls(unittest.TestCase):
class TestFlux3Lora(unittest.TestCase):
def test_rejects_late_lora_before_wrapping_prims_ts_weights(self):
_SUPPORTS_LORA = (
"sglang.multimodal_gen.runtime.pipelines.flux3_image."
"flux3_prims_ts_supports_lora"
)
@staticmethod
def _pipeline_with_one_prims_ts_layer():
from sglang.multimodal_gen.runtime.layers.linear import LinearBase
from sglang.multimodal_gen.runtime.layers.quantization.flux3_prims_ts import (
Flux3PrimsTsMethodMixin,
Flux3PrimsTsRole,
)
from sglang.multimodal_gen.runtime.pipelines.flux3_image import (
Flux3ImagePipeline,
@@ -635,12 +640,31 @@ class TestFlux3Lora(unittest.TestCase):
pipeline = Flux3ImagePipeline.__new__(Flux3ImagePipeline)
pipeline.modules = {"transformer": torch.nn.Sequential(layer)}
pipeline.lora_target_modules = None
pipeline._reject_lora_on_packed_weights()
layer.quant_method.prims_ts_role = Flux3PrimsTsRole.SWIGLU
with self.assertRaisesRegex(ValueError, "--lora-path"):
return pipeline, layer
def test_rejects_late_lora_before_wrapping_prims_ts_weights(self):
from sglang.multimodal_gen.runtime.layers.quantization.flux3_prims_ts import (
Flux3PrimsTsRole,
)
pipeline, layer = self._pipeline_with_one_prims_ts_layer()
with mock.patch(self._SUPPORTS_LORA, return_value=False):
pipeline._reject_lora_on_packed_weights()
layer.quant_method.prims_ts_role = Flux3PrimsTsRole.SWIGLU
with self.assertRaisesRegex(ValueError, "--lora-path"):
pipeline._reject_lora_on_packed_weights()
self.assertIs(pipeline.get_module("transformer")[0], layer)
def test_accepts_late_lora_when_prims_ts_takes_a_delta(self):
from sglang.multimodal_gen.runtime.layers.quantization.flux3_prims_ts import (
Flux3PrimsTsRole,
)
pipeline, layer = self._pipeline_with_one_prims_ts_layer()
layer.quant_method.prims_ts_role = Flux3PrimsTsRole.SWIGLU
with mock.patch(self._SUPPORTS_LORA, return_value=True):
pipeline._reject_lora_on_packed_weights()
def test_prefixed_adapter_keys_reach_the_native_modules(self):
"""Prefixed adapters (BFL/ostris ``dit.``, diffusers ``transformer.``)
must map onto the native DiT modules, not load onto 0 layers, and load
@@ -5,7 +5,8 @@ request scale, add) and only drops multiplications by 1.0, which are exact. At
"lossless" the second GEMM accumulates the scaled delta onto the base output and
rounds once, and a row-parallel bias moves into the base GEMM's epilogue. A
forward whose stage does not put its request in the forward context runs the
exact chain.
exact chain. ``fused_lora_delta`` returns the same delta for GEMM epilogues
that add it themselves.
"""
from types import SimpleNamespace
@@ -17,7 +18,10 @@ import torch.nn.functional as F
from torch import nn
from sglang.multimodal_gen.runtime.layers.lora.linear import (
BaseLayerWithLoRA,
ColumnParallelLinearWithLoRA,
LinearWithLoRA,
MergedColumnParallelLinearWithLoRA,
RowParallelLinearWithLoRA,
_compute_lora_delta,
_request_allows_lossless,
@@ -188,3 +192,109 @@ def test_row_parallel_bias_moves_into_the_gemm_only_above_exact():
candidate=lossless,
label="row-parallel LoRA with the bias in the GEMM",
)
class _ColumnLinear(nn.Module):
"""The ColumnParallelLinear surface the LoRA wrapper reads, at TP 1."""
def __init__(self, output_sizes: list[int]):
super().__init__()
g = torch.Generator().manual_seed(6)
self.weight = nn.Parameter(
torch.randn(sum(output_sizes), IN_DIM, generator=g) * 0.05
)
self.bias = None
self.output_sizes = output_sizes
self.output_partition_sizes = output_sizes
self.skip_bias_add = False
self.gather_output = False
self.quant_method = SimpleNamespace(
apply=lambda layer, x, bias=None: F.linear(x, layer.weight, bias)
)
def forward(self, x):
return F.linear(x, self.weight), None
def _column_layer(*, stacked=False, alpha=RANK, strength=1.0, output_offset=None):
a, b = _adapter(stacked, torch.float32)
if stacked:
layer = MergedColumnParallelLinearWithLoRA(
_ColumnLinear([OUT_DIM // 3] * 3), lora_rank=RANK, lora_alpha=alpha
)
else:
layer = ColumnParallelLinearWithLoRA(
_ColumnLinear([OUT_DIM]), lora_rank=RANK, lora_alpha=alpha
)
layer.set_lora_weights(
a, b, strength=strength, merge_weights=False, output_offset=output_offset
)
return layer
def _forward_delta(layer, x):
with patch(_TP_RANK_PATCH, return_value=0):
out, _ = layer(x)
base, _ = layer.base_layer(x)
return (out - base).reshape(-1, out.shape[-1])
def _fused_delta(layer, x, **kwargs):
with patch(_TP_RANK_PATCH, return_value=0):
return layer.fused_lora_delta(x, **kwargs)
def _assert_fused_delta(actual, expected):
assert actual.dtype == torch.bfloat16 and actual.is_contiguous()
torch.testing.assert_close(actual.float(), expected, rtol=2e-2, atol=2e-2)
@pytest.mark.parametrize("stacked", [False, True])
def test_fused_delta_matches_the_forward_delta(stacked):
layer = _column_layer(stacked=stacked, alpha=2 * RANK, strength=0.7)
x = _activations(TOKENS, IN_DIM, seed=7).reshape(2, TOKENS // 2, IN_DIM)
delta = _fused_delta(layer, x)
assert delta.shape == (TOKENS, OUT_DIM)
_assert_fused_delta(delta, _forward_delta(layer, x))
def test_fused_delta_follows_the_runtime_scale_and_output_offset():
layer = _column_layer(output_offset=torch.randn(OUT_DIM))
x = _activations(TOKENS, IN_DIM, seed=8)
with patch.object(BaseLayerWithLoRA, "_runtime_lora_scale", return_value=0.5):
_assert_fused_delta(_fused_delta(layer, x), _forward_delta(layer, x))
with patch.object(BaseLayerWithLoRA, "_runtime_lora_scale", return_value=0.0):
assert _fused_delta(layer, x) is None
def test_fused_delta_interleaves_swiglu_value_and_gate_columns():
layer = _column_layer(output_offset=torch.randn(OUT_DIM))
x = _activations(TOKENS, IN_DIM, seed=9)
gate, value = _forward_delta(layer, x).chunk(2, dim=-1)
interleaved = torch.stack((value, gate), dim=-1).flatten(-2)
_assert_fused_delta(_fused_delta(layer, x, swiglu_rows=True), interleaved)
def test_fused_delta_drops_cached_factors_for_new_adapter_weights():
layer = _column_layer()
x = _activations(TOKENS, IN_DIM, seed=10)
_fused_delta(layer, x)
a, b = _adapter(False, torch.float32, seed=11)
layer.set_lora_weights(a, b, merge_weights=False)
_assert_fused_delta(_fused_delta(layer, x), _forward_delta(layer, x))
@pytest.mark.parametrize("state", ["no_adapter", "disabled", "merged"])
def test_inactive_layers_have_no_fused_delta(state):
if state == "no_adapter":
layer = ColumnParallelLinearWithLoRA(
_ColumnLinear([OUT_DIM]), lora_rank=RANK, lora_alpha=RANK
)
else:
layer = _column_layer()
if state == "disabled":
layer.disable_lora = True
if state == "merged":
layer.merged = True
assert not layer.lora_active
assert _fused_delta(layer, _activations(TOKENS, IN_DIM, seed=12)) is None