[Perf] diffusion: fused gate/up + SwiGLU MLP on comfy_kitchen INT8 for Qwen-Image 2.1 (#44196)

This commit is contained in:
WenhaoZhang
2026-10-18 20:33:31 +08:00
committed by GitHub
parent 680a2ff27e
commit e17fed6ea7
3 changed files with 173 additions and 1 deletions
@@ -30,7 +30,7 @@ from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_conf
)
from sglang.multimodal_gen.runtime.utils.weight_attrs import set_weight_attrs
__all__ = ["ConvRotInt8ComfyKitchenLinearMethod"]
__all__ = ["ConvRotInt8ComfyKitchenLinearMethod", "comfy_kitchen_swiglu_mlp"]
# comfy_kitchen's dtype codes for the fused op's output.
_OUT_DTYPE_CODE = {torch.float32: 0, torch.float16: 1, torch.bfloat16: 2}
@@ -221,3 +221,37 @@ class ConvRotInt8ComfyKitchenLinearMethod(LinearMethodBase):
if len(orig_shape) != 2:
out = out.reshape(*orig_shape[:-1], out.shape[-1])
return out
def comfy_kitchen_swiglu_mlp(
x: torch.Tensor,
gate_up_weight: torch.Tensor,
gate_up_scale: torch.Tensor,
gate_up_group_size: int,
down: torch.nn.Module,
) -> torch.Tensor:
"""``down(silu(gate) * up)`` for ``[gate | up] = x @ gate_up_weight.T``, run as
ComfyUI runs SwiGLU MLPs: one GEMM over the stacked [gate; up] weight, and a
down-projection that applies the SwiGLU inside its quantizer
(``input_act="swiglu"``), so the activated rows never reach HBM.
The fused quantizer keeps ``silu(gate) * up`` in FP32 instead of rounding it
to BF16 first, so the result is not bitwise the eager form; it sits slightly
closer to an FP32 reference.
"""
out_code = _OUT_DTYPE_CODE[x.dtype]
rows = x.reshape(-1, x.shape[-1]).contiguous()
gate_up = torch.ops.comfy_kitchen.int8_linear(
rows, gate_up_weight, gate_up_scale, None, out_code, True, gate_up_group_size
)
out = torch.ops.comfy_kitchen.int8_linear(
gate_up,
down.weight,
down.weight_scale,
None,
out_code,
True, # convrot
down.quant_method.group_size,
"swiglu",
)
return out.reshape(*x.shape[:-1], out.shape[-1])
@@ -45,6 +45,10 @@ from sglang.multimodal_gen.runtime.layers.linear import (
UnquantizedLinearMethod,
)
from sglang.multimodal_gen.runtime.layers.lora.linear import BaseLayerWithLoRA
from sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_comfy_kitchen import (
ConvRotInt8ComfyKitchenLinearMethod,
comfy_kitchen_swiglu_mlp,
)
from sglang.multimodal_gen.runtime.managers.forward_context import get_forward_context
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
LayerwiseOffloadableModuleMixin,
@@ -318,6 +322,22 @@ class QwenImage21TimeEmbedding(nn.Module):
)
def _stacked_rows(a, b):
"""[a; b] as one view when ``b`` directly follows ``a`` in the same storage, else None."""
# Layerwise offload leaves a flat placeholder in place of each weight.
if not (
a.dim() == 2
and a.is_cuda
and a.shape == b.shape
and a.is_contiguous()
and b.is_contiguous()
and a.untyped_storage().data_ptr() == b.untyped_storage().data_ptr()
and b.storage_offset() == a.storage_offset() + a.numel()
):
return None
return a.as_strided((2 * a.shape[0], a.shape[1]), (a.shape[1], 1))
class QwenImage21FeedForward(nn.Module):
def __init__(self, dim, ratio, quant_config, prefix):
super().__init__()
@@ -343,7 +363,72 @@ class QwenImage21FeedForward(nn.Module):
prefix=f"{prefix}.out",
)
def pack_gate_up_weights(self):
"""Share one [2F, C] INT8 weight and [2F, 1] scale storage between gate_layer
and proj so the comfy_kitchen path projects both with one GEMM.
As with ``pack_qkv_weights``, parameters keep their names and shapes and
no reference to the shared storage is kept. Layers that are not plain
comfy_kitchen INT8 projections are left alone.
"""
gate, up = self.gate_layer, self.proj
if self.packed_gate_up() is not None:
return
if not (
all(
type(layer) is ColumnParallelLinear
and isinstance(layer.quant_method, ConvRotInt8ComfyKitchenLinearMethod)
and layer.weight.is_cuda
and layer.weight.dtype is torch.int8
and layer.weight.dim() == 2
for layer in (gate, up)
)
and gate.weight.shape == up.weight.shape
and gate.quant_method.group_size == up.quant_method.group_size
):
return
rows = gate.weight.shape[0]
weight = torch.cat([gate.weight.data, up.weight.data], dim=0)
scale = torch.cat([gate.weight_scale.data, up.weight_scale.data], dim=0)
gate.weight.data, up.weight.data = weight[:rows], weight[rows:]
gate.weight_scale.data, up.weight_scale.data = scale[:rows], scale[rows:]
def packed_gate_up(self):
"""The stacked [gate; up] weight and scale views when the comfy_kitchen
fused MLP can run, else None."""
gate, up, out = self.gate_layer, self.proj, self.out
# LoRA wrappers replace the layers; TP > 1 would need the down all-reduce.
if not (
type(gate) is ColumnParallelLinear
and type(up) is ColumnParallelLinear
and type(out) is RowParallelLinear
and out.tp_size == 1
and isinstance(gate.quant_method, ConvRotInt8ComfyKitchenLinearMethod)
and isinstance(out.quant_method, ConvRotInt8ComfyKitchenLinearMethod)
and out.weight.dtype is torch.int8
):
return None
weight = _stacked_rows(gate.weight, up.weight)
scale = _stacked_rows(gate.weight_scale, up.weight_scale)
if weight is None or scale is None:
return None
return weight, scale
def _apply(self, fn, recurse=True):
# .to() and CPU offload rebuild every parameter in its own storage;
# re-share them once the weights are back on the GPU.
module = super()._apply(fn, recurse)
self.pack_gate_up_weights()
return module
def forward(self, x):
# Storage-identity checks cannot be traced; a compiled graph keeps the
# separate projections.
packed = None if torch.compiler.is_compiling() else self.packed_gate_up()
if packed is not None:
return comfy_kitchen_swiglu_mlp(
x, *packed, self.gate_layer.quant_method.group_size, self.out
)
gate, value = self.gate_layer(x)[0], self.proj(x)[0]
fused = None
if (
@@ -917,6 +1002,7 @@ class QwenImage21Transformer2DModel(CachableDiT, LayerwiseOffloadableModuleMixin
super().post_load_weights()
for block in self.transformer_blocks:
block.attn.pack_qkv_weights()
block.img_mlp.pack_gate_up_weights()
def prepare_lora_adapter(self, adapter):
return _split_fused_gate_up_lora(
@@ -19,6 +19,7 @@ from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_conf
)
from sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_comfy_kitchen import (
ConvRotInt8ComfyKitchenLinearMethod,
comfy_kitchen_available,
)
from sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_jit import (
ConvRotInt8JitLinearMethod,
@@ -219,6 +220,57 @@ def test_gelu_input_helper_is_bitwise_eager_gelu_then_layer():
assert torch.equal(fused, eager)
@pytest.mark.skipif(
not (torch.cuda.is_available() and comfy_kitchen_available()),
reason="needs CUDA and comfy_kitchen",
)
def test_qwen_image21_kitchen_mlp_packs_gate_up_and_folds_swiglu(monkeypatch):
"""On comfy_kitchen the Qwen-Image 2.1 MLP runs ComfyUI's fused form; once
the shared gate/up storage is gone it falls back to separate projections."""
from sglang.multimodal_gen.runtime.models.dits.qwen_image21 import (
QwenImage21FeedForward,
)
_with_tp(monkeypatch, 1)
ffn = QwenImage21FeedForward(
1024, 3, ConvRotInt8Config(backend="comfy_kitchen"), "blocks.0.img_mlp"
).cuda()
gen = torch.Generator(device="cuda").manual_seed(0)
for layer in (ffn.gate_layer, ffn.proj, ffn.out):
with torch.no_grad():
layer.weight.copy_(
torch.randn(layer.weight.shape, device="cuda", generator=gen) * 0.02
)
out_weight = layer.weight.detach().float() # ends as ffn.out's BF16 weight
layer.quant_method.process_weights_after_loading(layer)
x = torch.randn(1, 2048, 1024, device="cuda", dtype=torch.bfloat16, generator=gen)
gate, up = ffn.gate_layer(x)[0], ffn.proj(x)[0]
eager = ffn.out(F.silu(gate) * up)[0]
ffn.pack_gate_up_weights()
assert ffn.packed_gate_up() is not None
assert ffn.gate_layer.weight.shape == ffn.proj.weight.shape == (3072, 1024)
fused = ffn(x)
# The fused quantizer keeps silu(gate) * up in FP32 rather than rounding it
# to BF16, so it is not bitwise the eager form; it must be no further (1%
# slack) from the FP32 down-projection of the same gate and up.
reference = (F.silu(gate.float()) * up.float()) @ out_weight.t()
def rel_l2(out):
err = torch.linalg.vector_norm(out.float() - reference)
return (err / torch.linalg.vector_norm(reference)).item()
assert rel_l2(fused) <= 1.01 * rel_l2(eager), (rel_l2(fused), rel_l2(eager))
# A CPU round trip rebuilds every parameter on its own; .cuda() re-packs.
ffn.cpu().cuda()
assert torch.equal(ffn(x), fused)
ffn.proj.weight.data = ffn.proj.weight.data.clone()
assert ffn.packed_gate_up() is None
assert torch.equal(ffn(x), eager)
class _FakeTpGroup:
pass