diff --git a/python/sglang/kernels/ops/diffusion/gemm/flux3_fp8_prims_ts.py b/python/sglang/kernels/ops/diffusion/gemm/flux3_fp8_prims_ts.py index a888bfd126..f155e5b6cd 100644 --- a/python/sglang/kernels/ops/diffusion/gemm/flux3_fp8_prims_ts.py +++ b/python/sglang/kernels/ops/diffusion/gemm/flux3_fp8_prims_ts.py @@ -9,6 +9,7 @@ import torch from flashinfer.prims_ts import gemm as prims_ts_gemm from sglang.kernels.ops.diffusion.gemm.flux3_prims_ts_config import ( + delta_kwargs, flux3_prims_ts_config, ) from sglang.kernels.ops.quantization.per_token_quant_fp8 import per_token_quant_fp8 @@ -90,6 +91,8 @@ def flux3_fused_mlp_up( packed_weight: torch.Tensor, packed_weight_scale: torch.Tensor, quantized: QuantizedActivation | None = None, + *, + delta: torch.Tensor | None = None, ) -> torch.Tensor: """Run fused FP8 GEMM + SwiGLU, quantizing BF16 rows when needed.""" if x.dtype != torch.bfloat16: @@ -103,6 +106,7 @@ def flux3_fused_mlp_up( config=_flux3_gemm_config( x, epilogue="swiglu", k=packed_weight.shape[1], n=packed_weight.shape[0] ), + **delta_kwargs(delta), skip_check=True, ) return out.view(*x.shape[:-1], out.shape[-1]) @@ -114,6 +118,8 @@ def flux3_clustered_fp8_linear( packed_weight_scale: torch.Tensor, quantized: QuantizedActivation | None = None, qkv_scale: torch.Tensor | None = None, + *, + delta: torch.Tensor | None = None, ) -> torch.Tensor: """Run a PrimsTS FP8 GEMM, optionally scaling packed Q/K/V thirds.""" if x.dtype != torch.bfloat16: @@ -132,6 +138,7 @@ def flux3_clustered_fp8_linear( n=packed_weight.shape[0], qkv_scale=qkv_scale is not None, ), + **delta_kwargs(delta), skip_check=True, ) return out.view(*x.shape[:-1], out.shape[-1]) @@ -149,6 +156,7 @@ def flux3_clustered_fp8_qkv_qknorm_rope( qkv_scale: torch.Tensor | None = None, *, separate_qkv_output: bool = False, + delta: torch.Tensor | None = None, ) -> torch.Tensor: """Fuse QKV GEMM, QK RMSNorm, and RoPE, quantizing rows when needed.""" if x.dtype != torch.bfloat16: @@ -194,6 +202,7 @@ def flux3_clustered_fp8_qkv_qknorm_rope( head_dim=q_norm_weight.numel(), qkv_scale=qkv_scale, config=config, + **delta_kwargs(delta), skip_check=True, ) if separate_qkv_output: diff --git a/python/sglang/kernels/ops/diffusion/gemm/flux3_nvfp4_prims_ts.py b/python/sglang/kernels/ops/diffusion/gemm/flux3_nvfp4_prims_ts.py index cbd5286974..dafc11c82f 100644 --- a/python/sglang/kernels/ops/diffusion/gemm/flux3_nvfp4_prims_ts.py +++ b/python/sglang/kernels/ops/diffusion/gemm/flux3_nvfp4_prims_ts.py @@ -16,6 +16,7 @@ from flashinfer import SfLayout, nvfp4_quantize from flashinfer.prims_ts import gemm as prims_ts_gemm from sglang.kernels.ops.diffusion.gemm.flux3_prims_ts_config import ( + delta_kwargs, flux3_prims_ts_config, ) @@ -253,6 +254,8 @@ def flux3_nvfp4_linear( input_global_scale: float, weight_global_scale: float, quantized: QuantizedNvfp4Activation | None = None, + *, + delta: torch.Tensor | None = None, ) -> torch.Tensor: """Run a FLUX.3 projection entirely through PrimsTS NVFP4 GEMM.""" a_packed, a_scale_128x4, _ = _quantized_activation(x, input_global_scale, quantized) @@ -270,7 +273,7 @@ def flux3_nvfp4_linear( ), ) if prepared is not None: - out = prepared(a_packed, a_scale_128x4) + out = prepared(a_packed, a_scale_128x4, **delta_kwargs(delta)) else: out = prims_ts_gemm.fp4_linear( a_packed, @@ -279,6 +282,7 @@ def flux3_nvfp4_linear( weight_packed, weight_scale_128x4, weight_global_scale, + **delta_kwargs(delta), skip_check=True, ) return out.view(*x.shape[:-1], out.shape[-1]) @@ -328,6 +332,8 @@ def flux3_nvfp4_linear_swiglu( input_global_scale: float, weight_global_scale: float, quantized: QuantizedNvfp4Activation | None = None, + *, + delta: torch.Tensor | None = None, ) -> torch.Tensor: """Run FLUX.3's gated MLP-up projection with PrimsTS' SwiGLU epilogue.""" a_packed, a_scale_128x4, _ = _quantized_activation(x, input_global_scale, quantized) @@ -347,7 +353,7 @@ def flux3_nvfp4_linear_swiglu( device=x.device, ) if prepared is not None: - out = prepared(a_packed, a_scale_128x4) + out = prepared(a_packed, a_scale_128x4, **delta_kwargs(delta)) else: out = prims_ts_gemm.fp4_linear_swiglu( a_packed, @@ -356,6 +362,7 @@ def flux3_nvfp4_linear_swiglu( weight_packed, weight_scale_128x4, weight_global_scale, + **delta_kwargs(delta), skip_check=True, ) return out.view(*x.shape[:-1], out.shape[-1]) @@ -425,6 +432,7 @@ def flux3_nvfp4_qkv_qknorm_rope( quantized: QuantizedNvfp4Activation | None = None, *, separate_qkv_output: bool = False, + delta: torch.Tensor | None = None, ) -> torch.Tensor: """Fuse NVFP4 QKV GEMM, Q/K RMSNorm, and RoPE through PrimsTS. @@ -458,6 +466,7 @@ def flux3_nvfp4_qkv_qknorm_rope( a_scale_128x4, cos_sin=cos_sin_cache, positions=positions, + **delta_kwargs(delta), ) elif separate_qkv_output: raise RuntimeError("separate QKV output requires a prepared CUDA projection") @@ -479,6 +488,7 @@ def flux3_nvfp4_qkv_qknorm_rope( head_dim=head_dim, is_neox=False, qkv_scale=qkv_scale, + **delta_kwargs(delta), skip_check=True, ) if separate_qkv_output: diff --git a/python/sglang/kernels/ops/diffusion/gemm/flux3_prims_ts_config.py b/python/sglang/kernels/ops/diffusion/gemm/flux3_prims_ts_config.py index 6c268dc92e..905baf5cc0 100644 --- a/python/sglang/kernels/ops/diffusion/gemm/flux3_prims_ts_config.py +++ b/python/sglang/kernels/ops/diffusion/gemm/flux3_prims_ts_config.py @@ -94,6 +94,11 @@ _TACTICS = { } +def delta_kwargs(delta: torch.Tensor | None) -> dict[str, torch.Tensor]: + # FlashInfer releases without the PrimsTS delta input reject the keyword. + return {} if delta is None else {"delta": delta} + + @functools.cache def _tmem_overlap_epilogues() -> frozenset[str]: # FlashInfer PR 6255 (SwiGLU) and 6286 (QKV) add tmem_overlap to these prepare diff --git a/python/sglang/multimodal_gen/runtime/layers/flux3_fused_ops.py b/python/sglang/multimodal_gen/runtime/layers/flux3_fused_ops.py index 34df6b3218..88bef517e6 100644 --- a/python/sglang/multimodal_gen/runtime/layers/flux3_fused_ops.py +++ b/python/sglang/multimodal_gen/runtime/layers/flux3_fused_ops.py @@ -8,6 +8,8 @@ plus SGLang's fused QK-norm/RoPE and SwiGLU kernels, on any device and format. FLUX.3 quant method (:mod:`...quantization.flux3_prims_ts`) prepared at load time. FlashInfer is selected per call, only on SM10x and only for layers whose method planned the matching role, so call sites never branch on the backend. +A LoRA-wrapped layer runs its base layer's PrimsTS epilogue with the adapter's +delta added before the activation, QK-norm, or output conversion. """ from __future__ import annotations @@ -18,15 +20,35 @@ from torch import nn from sglang.kernels.fused_op import BaseFusedOp, register_fused_op from sglang.kernels.spec import FormatSignature, KernelBackend from sglang.multimodal_gen.runtime.layers.layernorm import apply_qk_norm_rope +from sglang.multimodal_gen.runtime.layers.lora.linear import BaseLayerWithLoRA from sglang.multimodal_gen.runtime.layers.quantization.flux3_prims_ts import ( FLUX3_PRIMS_TS_CAPABILITIES, Flux3PrimsTsRole, ) +def _base_layer(layer: nn.Module) -> nn.Module: + """The quantized layer under an optional LoRA wrapper.""" + return layer.base_layer if isinstance(layer, BaseLayerWithLoRA) else layer + + +def _lora_active(layer: nn.Module) -> bool: + return isinstance(layer, BaseLayerWithLoRA) and layer.lora_active + + +def _lora_delta_kwargs( + layer: nn.Module, x: torch.Tensor, *, swiglu_rows: bool = False +) -> dict[str, torch.Tensor]: + """The ``delta`` kwarg while the LoRA adds something; old FlashInfer rejects it.""" + if not _lora_active(layer): + return {} + delta = layer.fused_lora_delta(x, swiglu_rows=swiglu_rows) + return {} if delta is None else {"delta": delta} + + def _prims_ts_method(layer: nn.Module, role: Flux3PrimsTsRole): """The layer's quant method if it planned ``role`` for PrimsTS, else None.""" - method = getattr(layer, "quant_method", None) + method = getattr(_base_layer(layer), "quant_method", None) if getattr(method, "prims_ts_role", None) is role: return method return None @@ -93,7 +115,10 @@ class Flux3LinearOp(_Flux3PrimsTsOp): return out def forward_flashinfer(self, layer, x, quantized=None) -> torch.Tensor: - return layer.quant_method.apply_prims_ts_linear(layer, x, quantized) + base = _base_layer(layer) + return base.quant_method.apply_prims_ts_linear( + base, x, quantized, **_lora_delta_kwargs(layer, x) + ) class Flux3QKVQKNormRopeOp(_Flux3PrimsTsOp): @@ -154,14 +179,16 @@ class Flux3QKVQKNormRopeOp(_Flux3PrimsTsOp): local_heads, quantized=None, ): - qkv = layer.quant_method.apply_prims_ts_qkv_qknorm_rope( - layer, + base = _base_layer(layer) + qkv = base.quant_method.apply_prims_ts_qkv_qknorm_rope( + base, x, norm.query_norm.weight, norm.key_norm.weight, cos_sin_cache, positions, quantized, + **_lora_delta_kwargs(layer, x), ) return _unpack_qkv(qkv, local_heads) @@ -185,17 +212,25 @@ class Flux3GatedMLPOp(_Flux3PrimsTsOp): return flux3_linear(mlp_out, _silu_and_mul(mlp)) def forward_flashinfer(self, mlp_in, mlp_out, x, quantized=None) -> torch.Tensor: - up = mlp_in.quant_method + up_layer = _base_layer(mlp_in) + up = up_layer.quant_method down = _prims_ts_method(mlp_out, Flux3PrimsTsRole.LINEAR) + up_delta = _lora_delta_kwargs(mlp_in, x, swiglu_rows=True) if ( down is not None + and not up_delta + and not _lora_active(mlp_out) and up.packed_activation_format is not None and down.packed_activation_format == up.packed_activation_format ): # Hand MLP-down the packed SwiGLU output without a BF16 round trip. - packed = up.apply_prims_ts_swiglu(mlp_in, x, quantized, packed_output=True) - return down.apply_prims_ts_linear_from_packed(mlp_out, packed) - return flux3_linear(mlp_out, up.apply_prims_ts_swiglu(mlp_in, x, quantized)) + packed = up.apply_prims_ts_swiglu( + up_layer, x, quantized, packed_output=True + ) + return down.apply_prims_ts_linear_from_packed(_base_layer(mlp_out), packed) + # A LoRA on MLP-down needs the BF16 SwiGLU output for its own delta. + hidden = up.apply_prims_ts_swiglu(up_layer, x, quantized, **up_delta) + return flux3_linear(mlp_out, hidden) _FLUX3_LINEAR = register_fused_op(Flux3LinearOp(), __name__, "_FLUX3_LINEAR") @@ -241,11 +276,11 @@ def flux3_shared_activation(x: torch.Tensor, qkv: nn.Module, mlp_in: nn.Module): if not x.is_cuda: return None up = _prims_ts_method(mlp_in, Flux3PrimsTsRole.SWIGLU) - qkv_method = getattr(qkv, "quant_method", None) + qkv_method = getattr(_base_layer(qkv), "quant_method", None) if ( up is None or getattr(qkv_method, "prims_ts_role", None) is None or qkv_method.activation_format != up.activation_format ): return None - return up.quantize_prims_ts_activation(mlp_in, x) + return up.quantize_prims_ts_activation(_base_layer(mlp_in), x) diff --git a/python/sglang/multimodal_gen/runtime/layers/lora/linear.py b/python/sglang/multimodal_gen/runtime/layers/lora/linear.py index 926a667d16..3bb2afd756 100644 --- a/python/sglang/multimodal_gen/runtime/layers/lora/linear.py +++ b/python/sglang/multimodal_gen/runtime/layers/lora/linear.py @@ -180,6 +180,7 @@ class BaseLayerWithLoRA(nn.Module): self.lora_B = None self.lora_output_offset = None self.has_lora_output_offset = False + self._fused_lora_cache: tuple[tuple, torch.Tensor, torch.Tensor] | None = None @property def weight(self): @@ -276,6 +277,86 @@ class BaseLayerWithLoRA(nn.Module): def slice_lora_b_weights(self, B: torch.Tensor) -> torch.Tensor: return B + @property + def lora_active(self) -> bool: + """Whether ``forward`` adds anything to the base layer's output.""" + if self.disable_lora: + return False + if self.merged: + return self.has_lora_output_offset + return self.lora_A is not None + + @staticmethod + def _interleave_swiglu_rows(t: torch.Tensor) -> torch.Tensor: + # [gate | value] rows -> [value_0, gate_0, value_1, gate_1, ...] + half = t.shape[0] // 2 + return torch.stack((t[half:], t[:half]), dim=1).reshape(t.shape) + + def _fused_lora_factors( + self, device: torch.device, swiglu_rows: bool + ) -> tuple[torch.Tensor, torch.Tensor]: + lora_A, lora_B = self.lora_A, self.lora_B + if isinstance(lora_B, DTensor): + lora_A, lora_B = lora_A.to_local(), lora_B.to_local() + key = ( + lora_A.data_ptr(), + lora_A._version, + lora_B.data_ptr(), + lora_B._version, + swiglu_rows, + device, + ) + cached = self._fused_lora_cache + if cached is not None and cached[0] == key: + return cached[1], cached[2] + A = self.slice_lora_a_weights(lora_A.to(device)) + B = self.slice_lora_b_weights(lora_B.to(device)) + if A.dim() == 3: + # Stacked groups: one shrink GEMM, then a block-diagonal expand. + A = A.reshape(-1, A.shape[-1]) + B = torch.block_diag(*B.unbind(0)) + if swiglu_rows: + B = self._interleave_swiglu_rows(B) + A, B = A.contiguous(), B.contiguous() + self._fused_lora_cache = (key, A, B) + return A, B + + def fused_lora_delta( + self, x: torch.Tensor, *, swiglu_rows: bool = False + ) -> torch.Tensor | None: + """``forward(x) - base_layer(x)`` as a contiguous BF16 ``[tokens, out]``. + + For GEMM epilogues that add the adapter's update themselves. None when + there is nothing to add. ``swiglu_rows`` orders the columns like the + interleaved FLUX.3 PrimsTS SwiGLU weight (``[value_i, gate_i]`` pairs). + """ + if not self.lora_active: + return None + runtime_lora_scale = 1.0 if self.merged else self._runtime_lora_scale() + if runtime_lora_scale == 0.0: + return None + x_2d = x.reshape(-1, x.shape[-1]) + delta = None + if not self.merged: + A, B = self._fused_lora_factors(device=x.device, swiglu_rows=swiglu_rows) + scale = self.strength * runtime_lora_scale + if self.lora_alpha != self.lora_rank: + scale *= self.lora_alpha / self.lora_rank # type: ignore + hidden = x_2d.to(dtype=A.dtype) @ A.T + if scale != 1.0: + hidden = hidden * scale + delta = hidden @ B.T + offset = self._active_lora_output_offset() + if offset is not None: + offset = offset.to(device=x.device) * runtime_lora_scale + if swiglu_rows: + offset = self._interleave_swiglu_rows(offset) + if delta is None: + delta = offset.expand(x_2d.shape[0], -1) + else: + delta = delta + offset.to(dtype=delta.dtype) + return delta.to(torch.bfloat16).contiguous() + def _scaled_lora_output_offset( self, offset: torch.Tensor | None, diff --git a/python/sglang/multimodal_gen/runtime/layers/quantization/flux3_prims_ts.py b/python/sglang/multimodal_gen/runtime/layers/quantization/flux3_prims_ts.py index 9fa89dcf3f..7a13fe34d5 100644 --- a/python/sglang/multimodal_gen/runtime/layers/quantization/flux3_prims_ts.py +++ b/python/sglang/multimodal_gen/runtime/layers/quantization/flux3_prims_ts.py @@ -11,6 +11,8 @@ Projections without a plan keep the generic method's layout and ``apply``. from __future__ import annotations +import functools +import inspect import logging from enum import Enum from typing import Any, ClassVar @@ -64,6 +66,25 @@ def flux3_prims_ts_available() -> bool: return capabilities_satisfied(FLUX3_PRIMS_TS_CAPABILITIES, PlatformInfo.detect()) +@functools.cache +def flux3_prims_ts_supports_lora() -> bool: + """Whether FlashInfer's PrimsTS GEMMs take the ``delta`` epilogue input.""" + try: + from flashinfer.prims_ts import gemm as prims_ts_gemm + except ImportError: + return False + entry_points = ( + prims_ts_gemm.fp8_linear, + prims_ts_gemm.fp8_linear_swiglu, + prims_ts_gemm.fp8_qkv_qknorm_rope, + prims_ts_gemm.fp4_linear, + prims_ts_gemm.fp4_linear_swiglu, + prims_ts_gemm.fp4_qkv_qknorm_rope, + prims_ts_gemm.PreparedFp4Linear.__call__, + ) + return all("delta" in inspect.signature(fn).parameters for fn in entry_points) + + def _use_separate_qkv_output() -> bool: """Whether TRT-LLM Gen Sage should receive directly separated Q/K/V.""" return envs.SGLANG_USE_SAGE_ATTENTION_IMPL.get().lower() in { @@ -180,7 +201,7 @@ class Flux3PrimsTsMethodMixin: raise NotImplementedError def apply_prims_ts_linear( - self, layer: nn.Module, x: torch.Tensor, quantized=None + self, layer: nn.Module, x: torch.Tensor, quantized=None, *, delta=None ) -> torch.Tensor: raise NotImplementedError @@ -196,6 +217,7 @@ class Flux3PrimsTsMethodMixin: quantized=None, *, packed_output: bool = False, + delta=None, ): raise NotImplementedError @@ -208,6 +230,8 @@ class Flux3PrimsTsMethodMixin: cos_sin_cache: torch.Tensor, positions: torch.Tensor, quantized=None, + *, + delta=None, ) -> torch.Tensor: raise NotImplementedError @@ -327,12 +351,12 @@ class Flux3Fp8LinearMethod(Flux3PrimsTsMethodMixin, Fp8LinearMethod): return quantize_flux3_activation(x) def apply_prims_ts_linear( - self, layer: nn.Module, x: torch.Tensor, quantized=None + self, layer: nn.Module, x: torch.Tensor, quantized=None, *, delta=None ) -> torch.Tensor: from sglang.kernels.ops.diffusion import flux3_clustered_fp8_linear return flux3_clustered_fp8_linear( - x, layer.weight.t(), layer.weight_scale, quantized + x, layer.weight.t(), layer.weight_scale, quantized, delta=delta ) def apply_prims_ts_swiglu( @@ -342,12 +366,15 @@ class Flux3Fp8LinearMethod(Flux3PrimsTsMethodMixin, Fp8LinearMethod): quantized=None, *, packed_output: bool = False, + delta=None, ) -> torch.Tensor: from sglang.kernels.ops.diffusion import flux3_fused_mlp_up if packed_output: raise NotImplementedError(f"{layer.prefix}: FP8 SwiGLU emits BF16 only") - return flux3_fused_mlp_up(x, layer.weight.t(), layer.weight_scale, quantized) + return flux3_fused_mlp_up( + x, layer.weight.t(), layer.weight_scale, quantized, delta=delta + ) def apply_prims_ts_qkv_qknorm_rope( self, @@ -358,6 +385,8 @@ class Flux3Fp8LinearMethod(Flux3PrimsTsMethodMixin, Fp8LinearMethod): cos_sin_cache: torch.Tensor, positions: torch.Tensor, quantized=None, + *, + delta=None, ) -> torch.Tensor: from sglang.kernels.ops.diffusion import ( flux3_clustered_fp8_qkv_qknorm_rope, @@ -373,6 +402,7 @@ class Flux3Fp8LinearMethod(Flux3PrimsTsMethodMixin, Fp8LinearMethod): positions, quantized, separate_qkv_output=_use_separate_qkv_output(), + delta=delta, ) @@ -594,7 +624,7 @@ class Flux3Nvfp4LinearMethod(Flux3PrimsTsMethodMixin, ModelOptFp4LinearMethod): return quantize_flux3_nvfp4_activation(x, layer.prims_ts_input_global_scale) def apply_prims_ts_linear( - self, layer: nn.Module, x: torch.Tensor, quantized=None + self, layer: nn.Module, x: torch.Tensor, quantized=None, *, delta=None ) -> torch.Tensor: from sglang.kernels.ops.diffusion import flux3_nvfp4_linear @@ -605,6 +635,7 @@ class Flux3Nvfp4LinearMethod(Flux3PrimsTsMethodMixin, ModelOptFp4LinearMethod): layer.prims_ts_input_global_scale, layer.prims_ts_weight_global_scale, quantized, + delta=delta, ) def apply_prims_ts_linear_from_packed( @@ -628,24 +659,34 @@ class Flux3Nvfp4LinearMethod(Flux3PrimsTsMethodMixin, ModelOptFp4LinearMethod): quantized=None, *, packed_output: bool = False, + delta=None, ): from sglang.kernels.ops.diffusion import ( flux3_nvfp4_linear_swiglu, flux3_nvfp4_linear_swiglu_fp4, ) - op = ( - flux3_nvfp4_linear_swiglu_fp4 - if packed_output - else flux3_nvfp4_linear_swiglu - ) - return op( + if packed_output: + if delta is not None: + raise NotImplementedError( + f"{layer.prefix}: packed NVFP4 SwiGLU output has no delta input" + ) + return flux3_nvfp4_linear_swiglu_fp4( + x, + layer.weight, + layer.weight_scale_prims_ts_128x4, + layer.prims_ts_input_global_scale, + layer.prims_ts_weight_global_scale, + quantized, + ) + return flux3_nvfp4_linear_swiglu( x, layer.weight, layer.weight_scale_prims_ts_128x4, layer.prims_ts_input_global_scale, layer.prims_ts_weight_global_scale, quantized, + delta=delta, ) def apply_prims_ts_qkv_qknorm_rope( @@ -657,6 +698,8 @@ class Flux3Nvfp4LinearMethod(Flux3PrimsTsMethodMixin, ModelOptFp4LinearMethod): cos_sin_cache: torch.Tensor, positions: torch.Tensor, quantized=None, + *, + delta=None, ) -> torch.Tensor: from sglang.kernels.ops.diffusion import ( flux3_nvfp4_qkv_qknorm_rope, @@ -675,4 +718,5 @@ class Flux3Nvfp4LinearMethod(Flux3PrimsTsMethodMixin, ModelOptFp4LinearMethod): getattr(layer, "prims_ts_qkv_scale", None), quantized, separate_qkv_output=_use_separate_qkv_output(), + delta=delta, ) diff --git a/python/sglang/multimodal_gen/runtime/pipelines/flux3_image.py b/python/sglang/multimodal_gen/runtime/pipelines/flux3_image.py index 2022afc300..28a1f585b0 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines/flux3_image.py +++ b/python/sglang/multimodal_gen/runtime/pipelines/flux3_image.py @@ -14,6 +14,7 @@ from sglang.multimodal_gen.runtime.distributed import get_local_torch_device from sglang.multimodal_gen.runtime.layers.linear import LinearBase from sglang.multimodal_gen.runtime.layers.quantization.flux3_prims_ts import ( Flux3PrimsTsMethodMixin, + flux3_prims_ts_supports_lora, ) from sglang.multimodal_gen.runtime.loader.utils import ( get_memory_usage_of_component, @@ -53,6 +54,8 @@ class Flux3ImagePipeline(LoRAPipeline, ComposedPipelineBase): def _reject_lora_on_packed_weights(self) -> None: super()._reject_lora_on_packed_weights() + if flux3_prims_ts_supports_lora(): + return for layer in self.get_module("transformer").modules(): if ( isinstance(layer, LinearBase) @@ -60,7 +63,8 @@ class Flux3ImagePipeline(LoRAPipeline, ComposedPipelineBase): and layer.quant_method.prims_ts_role is not None ): raise ValueError( - "Dynamic LoRA cannot wrap PrimsTS-prepared FLUX.3 weights. " + "Dynamic LoRA cannot wrap PrimsTS-prepared FLUX.3 weights: the " + "installed FlashInfer cannot add a LoRA delta to PrimsTS GEMMs. " "Start with --lora-path to load the generic quantized layout." ) @@ -86,11 +90,13 @@ class Flux3ImagePipeline(LoRAPipeline, ComposedPipelineBase): device = get_local_torch_device() cpu = torch.device("cpu") quant_config = resolve_flux3_image_package_quant_config(package) - enable_prims_ts = server_args.lora_path is None + enable_prims_ts = ( + server_args.lora_path is None or flux3_prims_ts_supports_lora() + ) if not enable_prims_ts and quant_config is not None: logger.info( - "FLUX.3 PrimsTS is disabled because dynamic LoRA requires " - "the generic quantized linear layout." + "FLUX.3 PrimsTS is disabled because the installed FlashInfer " + "cannot add a dynamic LoRA delta in its GEMM epilogue." ) dit = load_flux3_image_dit( str(package.weights), diff --git a/python/sglang/multimodal_gen/test/unit/test_flux3_image_dit.py b/python/sglang/multimodal_gen/test/unit/test_flux3_image_dit.py index 1d023c93cb..6f331c0e8e 100644 --- a/python/sglang/multimodal_gen/test/unit/test_flux3_image_dit.py +++ b/python/sglang/multimodal_gen/test/unit/test_flux3_image_dit.py @@ -25,6 +25,7 @@ from sglang.multimodal_gen.runtime.layers.flux3_fused_ops import ( flux3_qkv_qknorm_rope, flux3_shared_activation, ) +from sglang.multimodal_gen.runtime.layers.lora.linear import BaseLayerWithLoRA from sglang.multimodal_gen.runtime.layers.quantization import flux3_prims_ts from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import ( ConvRotInt8Config, @@ -430,6 +431,15 @@ class _StubLinear: return self.output, None +def _stub_lora(base_layer, delta): + """A LoRA wrapper around a FLUX.3 linear whose adapter adds ``delta``.""" + stub = mock.NonCallableMock(spec=BaseLayerWithLoRA) + stub.base_layer = base_layer + stub.lora_active = True + stub.fused_lora_delta.return_value = delta + return stub + + class TestFlux3FusedOps(unittest.TestCase): _CUDA_X = SimpleNamespace(is_cuda=True) @@ -531,6 +541,71 @@ class TestFlux3FusedOps(unittest.TestCase): swiglu.assert_called_with(mlp_in, self._CUDA_X, "shared", packed_output=True) from_packed.assert_called_once_with(mlp_out, hidden) + def test_lora_wrapped_layers_pass_their_delta_to_the_prims_ts_epilogue(self): + x = self._CUDA_X + linear = mock.Mock(return_value="prims_ts") + planned = _StubLinear(Flux3PrimsTsRole.LINEAR, apply_prims_ts_linear=linear) + epilogue = mock.Mock(return_value=torch.randn(1, 3, 3 * 2 * 4)) + qkv = _StubLinear( + Flux3PrimsTsRole.QKV_QKNORM_ROPE, apply_prims_ts_qkv_qknorm_rope=epilogue + ) + norm = SimpleNamespace( + query_norm=SimpleNamespace(weight="q_w"), + key_norm=SimpleNamespace(weight="k_w"), + ) + wrapped = _stub_lora(planned, "delta") + with self._platform(10, 0): + self.assertEqual(flux3_linear(wrapped, x, "shared"), "prims_ts") + flux3_qkv_qknorm_rope(_stub_lora(qkv, "qkv_delta"), x, norm, "c", "p", 2) + # A zero runtime LoRA scale adds nothing: the plain kernel runs. + flux3_linear(_stub_lora(planned, None), x) + wrapped.fused_lora_delta.assert_called_once_with(x, swiglu_rows=False) + self.assertEqual( + linear.call_args_list, + [ + mock.call(planned, x, "shared", delta="delta"), + mock.call(planned, x, None), + ], + ) + epilogue.assert_called_once_with( + qkv, x, "q_w", "k_w", "c", "p", None, delta="qkv_delta" + ) + + def test_gated_mlp_with_lora_skips_the_packed_handoff(self): + """Either projection's LoRA needs the BF16 SwiGLU output: MLP-up adds its + delta in SwiGLU column order, MLP-down computes its delta from it.""" + x, hidden = self._CUDA_X, SimpleNamespace(is_cuda=True) + swiglu = mock.Mock(return_value=hidden) + down = mock.Mock(return_value="down") + from_packed = mock.Mock(return_value="down_from_packed") + mlp_in = _StubLinear(Flux3PrimsTsRole.SWIGLU, apply_prims_ts_swiglu=swiglu) + mlp_out = _StubLinear( + Flux3PrimsTsRole.LINEAR, + apply_prims_ts_linear=down, + apply_prims_ts_linear_from_packed=from_packed, + ) + for layer in (mlp_in, mlp_out): + layer.quant_method.packed_activation_format = "nvfp4" + up_lora = _stub_lora(mlp_in, "up_delta") + down_lora = _stub_lora(mlp_out, "down_delta") + with self._platform(10, 0): + self.assertEqual(flux3_gated_mlp(up_lora, mlp_out, x), "down") + self.assertEqual(flux3_gated_mlp(mlp_in, down_lora, x), "down") + up_lora.fused_lora_delta.assert_called_once_with(x, swiglu_rows=True) + down_lora.fused_lora_delta.assert_called_once_with(hidden, swiglu_rows=False) + self.assertEqual( + swiglu.call_args_list, + [mock.call(mlp_in, x, None, delta="up_delta"), mock.call(mlp_in, x, None)], + ) + self.assertEqual( + down.call_args_list, + [ + mock.call(mlp_out, hidden, None), + mock.call(mlp_out, hidden, None, delta="down_delta"), + ], + ) + from_packed.assert_not_called() + def test_shared_activation_requires_matching_prims_ts_consumers(self): x = self._CUDA_X quantize = mock.Mock(return_value="shared") diff --git a/python/sglang/multimodal_gen/test/unit/test_flux3_image_served.py b/python/sglang/multimodal_gen/test/unit/test_flux3_image_served.py index 4ba743230f..e4d4fe3e4a 100644 --- a/python/sglang/multimodal_gen/test/unit/test_flux3_image_served.py +++ b/python/sglang/multimodal_gen/test/unit/test_flux3_image_served.py @@ -619,11 +619,16 @@ class TestFlux3RequestControls(unittest.TestCase): class TestFlux3Lora(unittest.TestCase): - def test_rejects_late_lora_before_wrapping_prims_ts_weights(self): + _SUPPORTS_LORA = ( + "sglang.multimodal_gen.runtime.pipelines.flux3_image." + "flux3_prims_ts_supports_lora" + ) + + @staticmethod + def _pipeline_with_one_prims_ts_layer(): from sglang.multimodal_gen.runtime.layers.linear import LinearBase from sglang.multimodal_gen.runtime.layers.quantization.flux3_prims_ts import ( Flux3PrimsTsMethodMixin, - Flux3PrimsTsRole, ) from sglang.multimodal_gen.runtime.pipelines.flux3_image import ( Flux3ImagePipeline, @@ -635,12 +640,31 @@ class TestFlux3Lora(unittest.TestCase): pipeline = Flux3ImagePipeline.__new__(Flux3ImagePipeline) pipeline.modules = {"transformer": torch.nn.Sequential(layer)} pipeline.lora_target_modules = None - pipeline._reject_lora_on_packed_weights() - layer.quant_method.prims_ts_role = Flux3PrimsTsRole.SWIGLU - with self.assertRaisesRegex(ValueError, "--lora-path"): + return pipeline, layer + + def test_rejects_late_lora_before_wrapping_prims_ts_weights(self): + from sglang.multimodal_gen.runtime.layers.quantization.flux3_prims_ts import ( + Flux3PrimsTsRole, + ) + + pipeline, layer = self._pipeline_with_one_prims_ts_layer() + with mock.patch(self._SUPPORTS_LORA, return_value=False): pipeline._reject_lora_on_packed_weights() + layer.quant_method.prims_ts_role = Flux3PrimsTsRole.SWIGLU + with self.assertRaisesRegex(ValueError, "--lora-path"): + pipeline._reject_lora_on_packed_weights() self.assertIs(pipeline.get_module("transformer")[0], layer) + def test_accepts_late_lora_when_prims_ts_takes_a_delta(self): + from sglang.multimodal_gen.runtime.layers.quantization.flux3_prims_ts import ( + Flux3PrimsTsRole, + ) + + pipeline, layer = self._pipeline_with_one_prims_ts_layer() + layer.quant_method.prims_ts_role = Flux3PrimsTsRole.SWIGLU + with mock.patch(self._SUPPORTS_LORA, return_value=True): + pipeline._reject_lora_on_packed_weights() + def test_prefixed_adapter_keys_reach_the_native_modules(self): """Prefixed adapters (BFL/ostris ``dit.``, diffusers ``transformer.``) must map onto the native DiT modules, not load onto 0 layers, and load diff --git a/python/sglang/multimodal_gen/test/unit/test_lora_dynamic_delta_in_gemm.py b/python/sglang/multimodal_gen/test/unit/test_lora_dynamic_delta_in_gemm.py index 35afa09483..7b98674c80 100644 --- a/python/sglang/multimodal_gen/test/unit/test_lora_dynamic_delta_in_gemm.py +++ b/python/sglang/multimodal_gen/test/unit/test_lora_dynamic_delta_in_gemm.py @@ -5,7 +5,8 @@ request scale, add) and only drops multiplications by 1.0, which are exact. At "lossless" the second GEMM accumulates the scaled delta onto the base output and rounds once, and a row-parallel bias moves into the base GEMM's epilogue. A forward whose stage does not put its request in the forward context runs the -exact chain. +exact chain. ``fused_lora_delta`` returns the same delta for GEMM epilogues +that add it themselves. """ from types import SimpleNamespace @@ -17,7 +18,10 @@ import torch.nn.functional as F from torch import nn from sglang.multimodal_gen.runtime.layers.lora.linear import ( + BaseLayerWithLoRA, + ColumnParallelLinearWithLoRA, LinearWithLoRA, + MergedColumnParallelLinearWithLoRA, RowParallelLinearWithLoRA, _compute_lora_delta, _request_allows_lossless, @@ -188,3 +192,109 @@ def test_row_parallel_bias_moves_into_the_gemm_only_above_exact(): candidate=lossless, label="row-parallel LoRA with the bias in the GEMM", ) + + +class _ColumnLinear(nn.Module): + """The ColumnParallelLinear surface the LoRA wrapper reads, at TP 1.""" + + def __init__(self, output_sizes: list[int]): + super().__init__() + g = torch.Generator().manual_seed(6) + self.weight = nn.Parameter( + torch.randn(sum(output_sizes), IN_DIM, generator=g) * 0.05 + ) + self.bias = None + self.output_sizes = output_sizes + self.output_partition_sizes = output_sizes + self.skip_bias_add = False + self.gather_output = False + self.quant_method = SimpleNamespace( + apply=lambda layer, x, bias=None: F.linear(x, layer.weight, bias) + ) + + def forward(self, x): + return F.linear(x, self.weight), None + + +def _column_layer(*, stacked=False, alpha=RANK, strength=1.0, output_offset=None): + a, b = _adapter(stacked, torch.float32) + if stacked: + layer = MergedColumnParallelLinearWithLoRA( + _ColumnLinear([OUT_DIM // 3] * 3), lora_rank=RANK, lora_alpha=alpha + ) + else: + layer = ColumnParallelLinearWithLoRA( + _ColumnLinear([OUT_DIM]), lora_rank=RANK, lora_alpha=alpha + ) + layer.set_lora_weights( + a, b, strength=strength, merge_weights=False, output_offset=output_offset + ) + return layer + + +def _forward_delta(layer, x): + with patch(_TP_RANK_PATCH, return_value=0): + out, _ = layer(x) + base, _ = layer.base_layer(x) + return (out - base).reshape(-1, out.shape[-1]) + + +def _fused_delta(layer, x, **kwargs): + with patch(_TP_RANK_PATCH, return_value=0): + return layer.fused_lora_delta(x, **kwargs) + + +def _assert_fused_delta(actual, expected): + assert actual.dtype == torch.bfloat16 and actual.is_contiguous() + torch.testing.assert_close(actual.float(), expected, rtol=2e-2, atol=2e-2) + + +@pytest.mark.parametrize("stacked", [False, True]) +def test_fused_delta_matches_the_forward_delta(stacked): + layer = _column_layer(stacked=stacked, alpha=2 * RANK, strength=0.7) + x = _activations(TOKENS, IN_DIM, seed=7).reshape(2, TOKENS // 2, IN_DIM) + delta = _fused_delta(layer, x) + assert delta.shape == (TOKENS, OUT_DIM) + _assert_fused_delta(delta, _forward_delta(layer, x)) + + +def test_fused_delta_follows_the_runtime_scale_and_output_offset(): + layer = _column_layer(output_offset=torch.randn(OUT_DIM)) + x = _activations(TOKENS, IN_DIM, seed=8) + with patch.object(BaseLayerWithLoRA, "_runtime_lora_scale", return_value=0.5): + _assert_fused_delta(_fused_delta(layer, x), _forward_delta(layer, x)) + with patch.object(BaseLayerWithLoRA, "_runtime_lora_scale", return_value=0.0): + assert _fused_delta(layer, x) is None + + +def test_fused_delta_interleaves_swiglu_value_and_gate_columns(): + layer = _column_layer(output_offset=torch.randn(OUT_DIM)) + x = _activations(TOKENS, IN_DIM, seed=9) + gate, value = _forward_delta(layer, x).chunk(2, dim=-1) + interleaved = torch.stack((value, gate), dim=-1).flatten(-2) + _assert_fused_delta(_fused_delta(layer, x, swiglu_rows=True), interleaved) + + +def test_fused_delta_drops_cached_factors_for_new_adapter_weights(): + layer = _column_layer() + x = _activations(TOKENS, IN_DIM, seed=10) + _fused_delta(layer, x) + a, b = _adapter(False, torch.float32, seed=11) + layer.set_lora_weights(a, b, merge_weights=False) + _assert_fused_delta(_fused_delta(layer, x), _forward_delta(layer, x)) + + +@pytest.mark.parametrize("state", ["no_adapter", "disabled", "merged"]) +def test_inactive_layers_have_no_fused_delta(state): + if state == "no_adapter": + layer = ColumnParallelLinearWithLoRA( + _ColumnLinear([OUT_DIM]), lora_rank=RANK, lora_alpha=RANK + ) + else: + layer = _column_layer() + if state == "disabled": + layer.disable_lora = True + if state == "merged": + layer.merged = True + assert not layer.lora_active + assert _fused_delta(layer, _activations(TOKENS, IN_DIM, seed=12)) is None