[Perf] diffusion: fused gate/up + SwiGLU MLP on comfy_kitchen INT8 for Qwen-Image 2.1 (#44196)
This commit is contained in:
+35
-1
@@ -30,7 +30,7 @@ from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_conf
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.utils.weight_attrs import set_weight_attrs
|
||||
|
||||
__all__ = ["ConvRotInt8ComfyKitchenLinearMethod"]
|
||||
__all__ = ["ConvRotInt8ComfyKitchenLinearMethod", "comfy_kitchen_swiglu_mlp"]
|
||||
|
||||
# comfy_kitchen's dtype codes for the fused op's output.
|
||||
_OUT_DTYPE_CODE = {torch.float32: 0, torch.float16: 1, torch.bfloat16: 2}
|
||||
@@ -221,3 +221,37 @@ class ConvRotInt8ComfyKitchenLinearMethod(LinearMethodBase):
|
||||
if len(orig_shape) != 2:
|
||||
out = out.reshape(*orig_shape[:-1], out.shape[-1])
|
||||
return out
|
||||
|
||||
|
||||
def comfy_kitchen_swiglu_mlp(
|
||||
x: torch.Tensor,
|
||||
gate_up_weight: torch.Tensor,
|
||||
gate_up_scale: torch.Tensor,
|
||||
gate_up_group_size: int,
|
||||
down: torch.nn.Module,
|
||||
) -> torch.Tensor:
|
||||
"""``down(silu(gate) * up)`` for ``[gate | up] = x @ gate_up_weight.T``, run as
|
||||
ComfyUI runs SwiGLU MLPs: one GEMM over the stacked [gate; up] weight, and a
|
||||
down-projection that applies the SwiGLU inside its quantizer
|
||||
(``input_act="swiglu"``), so the activated rows never reach HBM.
|
||||
|
||||
The fused quantizer keeps ``silu(gate) * up`` in FP32 instead of rounding it
|
||||
to BF16 first, so the result is not bitwise the eager form; it sits slightly
|
||||
closer to an FP32 reference.
|
||||
"""
|
||||
out_code = _OUT_DTYPE_CODE[x.dtype]
|
||||
rows = x.reshape(-1, x.shape[-1]).contiguous()
|
||||
gate_up = torch.ops.comfy_kitchen.int8_linear(
|
||||
rows, gate_up_weight, gate_up_scale, None, out_code, True, gate_up_group_size
|
||||
)
|
||||
out = torch.ops.comfy_kitchen.int8_linear(
|
||||
gate_up,
|
||||
down.weight,
|
||||
down.weight_scale,
|
||||
None,
|
||||
out_code,
|
||||
True, # convrot
|
||||
down.quant_method.group_size,
|
||||
"swiglu",
|
||||
)
|
||||
return out.reshape(*x.shape[:-1], out.shape[-1])
|
||||
|
||||
@@ -45,6 +45,10 @@ from sglang.multimodal_gen.runtime.layers.linear import (
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.lora.linear import BaseLayerWithLoRA
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_comfy_kitchen import (
|
||||
ConvRotInt8ComfyKitchenLinearMethod,
|
||||
comfy_kitchen_swiglu_mlp,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.forward_context import get_forward_context
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
|
||||
LayerwiseOffloadableModuleMixin,
|
||||
@@ -318,6 +322,22 @@ class QwenImage21TimeEmbedding(nn.Module):
|
||||
)
|
||||
|
||||
|
||||
def _stacked_rows(a, b):
|
||||
"""[a; b] as one view when ``b`` directly follows ``a`` in the same storage, else None."""
|
||||
# Layerwise offload leaves a flat placeholder in place of each weight.
|
||||
if not (
|
||||
a.dim() == 2
|
||||
and a.is_cuda
|
||||
and a.shape == b.shape
|
||||
and a.is_contiguous()
|
||||
and b.is_contiguous()
|
||||
and a.untyped_storage().data_ptr() == b.untyped_storage().data_ptr()
|
||||
and b.storage_offset() == a.storage_offset() + a.numel()
|
||||
):
|
||||
return None
|
||||
return a.as_strided((2 * a.shape[0], a.shape[1]), (a.shape[1], 1))
|
||||
|
||||
|
||||
class QwenImage21FeedForward(nn.Module):
|
||||
def __init__(self, dim, ratio, quant_config, prefix):
|
||||
super().__init__()
|
||||
@@ -343,7 +363,72 @@ class QwenImage21FeedForward(nn.Module):
|
||||
prefix=f"{prefix}.out",
|
||||
)
|
||||
|
||||
def pack_gate_up_weights(self):
|
||||
"""Share one [2F, C] INT8 weight and [2F, 1] scale storage between gate_layer
|
||||
and proj so the comfy_kitchen path projects both with one GEMM.
|
||||
|
||||
As with ``pack_qkv_weights``, parameters keep their names and shapes and
|
||||
no reference to the shared storage is kept. Layers that are not plain
|
||||
comfy_kitchen INT8 projections are left alone.
|
||||
"""
|
||||
gate, up = self.gate_layer, self.proj
|
||||
if self.packed_gate_up() is not None:
|
||||
return
|
||||
if not (
|
||||
all(
|
||||
type(layer) is ColumnParallelLinear
|
||||
and isinstance(layer.quant_method, ConvRotInt8ComfyKitchenLinearMethod)
|
||||
and layer.weight.is_cuda
|
||||
and layer.weight.dtype is torch.int8
|
||||
and layer.weight.dim() == 2
|
||||
for layer in (gate, up)
|
||||
)
|
||||
and gate.weight.shape == up.weight.shape
|
||||
and gate.quant_method.group_size == up.quant_method.group_size
|
||||
):
|
||||
return
|
||||
rows = gate.weight.shape[0]
|
||||
weight = torch.cat([gate.weight.data, up.weight.data], dim=0)
|
||||
scale = torch.cat([gate.weight_scale.data, up.weight_scale.data], dim=0)
|
||||
gate.weight.data, up.weight.data = weight[:rows], weight[rows:]
|
||||
gate.weight_scale.data, up.weight_scale.data = scale[:rows], scale[rows:]
|
||||
|
||||
def packed_gate_up(self):
|
||||
"""The stacked [gate; up] weight and scale views when the comfy_kitchen
|
||||
fused MLP can run, else None."""
|
||||
gate, up, out = self.gate_layer, self.proj, self.out
|
||||
# LoRA wrappers replace the layers; TP > 1 would need the down all-reduce.
|
||||
if not (
|
||||
type(gate) is ColumnParallelLinear
|
||||
and type(up) is ColumnParallelLinear
|
||||
and type(out) is RowParallelLinear
|
||||
and out.tp_size == 1
|
||||
and isinstance(gate.quant_method, ConvRotInt8ComfyKitchenLinearMethod)
|
||||
and isinstance(out.quant_method, ConvRotInt8ComfyKitchenLinearMethod)
|
||||
and out.weight.dtype is torch.int8
|
||||
):
|
||||
return None
|
||||
weight = _stacked_rows(gate.weight, up.weight)
|
||||
scale = _stacked_rows(gate.weight_scale, up.weight_scale)
|
||||
if weight is None or scale is None:
|
||||
return None
|
||||
return weight, scale
|
||||
|
||||
def _apply(self, fn, recurse=True):
|
||||
# .to() and CPU offload rebuild every parameter in its own storage;
|
||||
# re-share them once the weights are back on the GPU.
|
||||
module = super()._apply(fn, recurse)
|
||||
self.pack_gate_up_weights()
|
||||
return module
|
||||
|
||||
def forward(self, x):
|
||||
# Storage-identity checks cannot be traced; a compiled graph keeps the
|
||||
# separate projections.
|
||||
packed = None if torch.compiler.is_compiling() else self.packed_gate_up()
|
||||
if packed is not None:
|
||||
return comfy_kitchen_swiglu_mlp(
|
||||
x, *packed, self.gate_layer.quant_method.group_size, self.out
|
||||
)
|
||||
gate, value = self.gate_layer(x)[0], self.proj(x)[0]
|
||||
fused = None
|
||||
if (
|
||||
@@ -917,6 +1002,7 @@ class QwenImage21Transformer2DModel(CachableDiT, LayerwiseOffloadableModuleMixin
|
||||
super().post_load_weights()
|
||||
for block in self.transformer_blocks:
|
||||
block.attn.pack_qkv_weights()
|
||||
block.img_mlp.pack_gate_up_weights()
|
||||
|
||||
def prepare_lora_adapter(self, adapter):
|
||||
return _split_fused_gate_up_lora(
|
||||
|
||||
@@ -19,6 +19,7 @@ from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_conf
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_comfy_kitchen import (
|
||||
ConvRotInt8ComfyKitchenLinearMethod,
|
||||
comfy_kitchen_available,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_jit import (
|
||||
ConvRotInt8JitLinearMethod,
|
||||
@@ -219,6 +220,57 @@ def test_gelu_input_helper_is_bitwise_eager_gelu_then_layer():
|
||||
assert torch.equal(fused, eager)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not (torch.cuda.is_available() and comfy_kitchen_available()),
|
||||
reason="needs CUDA and comfy_kitchen",
|
||||
)
|
||||
def test_qwen_image21_kitchen_mlp_packs_gate_up_and_folds_swiglu(monkeypatch):
|
||||
"""On comfy_kitchen the Qwen-Image 2.1 MLP runs ComfyUI's fused form; once
|
||||
the shared gate/up storage is gone it falls back to separate projections."""
|
||||
from sglang.multimodal_gen.runtime.models.dits.qwen_image21 import (
|
||||
QwenImage21FeedForward,
|
||||
)
|
||||
|
||||
_with_tp(monkeypatch, 1)
|
||||
ffn = QwenImage21FeedForward(
|
||||
1024, 3, ConvRotInt8Config(backend="comfy_kitchen"), "blocks.0.img_mlp"
|
||||
).cuda()
|
||||
gen = torch.Generator(device="cuda").manual_seed(0)
|
||||
for layer in (ffn.gate_layer, ffn.proj, ffn.out):
|
||||
with torch.no_grad():
|
||||
layer.weight.copy_(
|
||||
torch.randn(layer.weight.shape, device="cuda", generator=gen) * 0.02
|
||||
)
|
||||
out_weight = layer.weight.detach().float() # ends as ffn.out's BF16 weight
|
||||
layer.quant_method.process_weights_after_loading(layer)
|
||||
x = torch.randn(1, 2048, 1024, device="cuda", dtype=torch.bfloat16, generator=gen)
|
||||
gate, up = ffn.gate_layer(x)[0], ffn.proj(x)[0]
|
||||
eager = ffn.out(F.silu(gate) * up)[0]
|
||||
|
||||
ffn.pack_gate_up_weights()
|
||||
assert ffn.packed_gate_up() is not None
|
||||
assert ffn.gate_layer.weight.shape == ffn.proj.weight.shape == (3072, 1024)
|
||||
fused = ffn(x)
|
||||
# The fused quantizer keeps silu(gate) * up in FP32 rather than rounding it
|
||||
# to BF16, so it is not bitwise the eager form; it must be no further (1%
|
||||
# slack) from the FP32 down-projection of the same gate and up.
|
||||
reference = (F.silu(gate.float()) * up.float()) @ out_weight.t()
|
||||
|
||||
def rel_l2(out):
|
||||
err = torch.linalg.vector_norm(out.float() - reference)
|
||||
return (err / torch.linalg.vector_norm(reference)).item()
|
||||
|
||||
assert rel_l2(fused) <= 1.01 * rel_l2(eager), (rel_l2(fused), rel_l2(eager))
|
||||
|
||||
# A CPU round trip rebuilds every parameter on its own; .cuda() re-packs.
|
||||
ffn.cpu().cuda()
|
||||
assert torch.equal(ffn(x), fused)
|
||||
|
||||
ffn.proj.weight.data = ffn.proj.weight.data.clone()
|
||||
assert ffn.packed_gate_up() is None
|
||||
assert torch.equal(ffn(x), eager)
|
||||
|
||||
|
||||
class _FakeTpGroup:
|
||||
pass
|
||||
|
||||
|
||||
Reference in New Issue
Block a user