diff --git a/python/sglang/multimodal_gen/runtime/layers/quantization/convrot_int8_comfy_kitchen.py b/python/sglang/multimodal_gen/runtime/layers/quantization/convrot_int8_comfy_kitchen.py index 435893910c..98c82544ae 100644 --- a/python/sglang/multimodal_gen/runtime/layers/quantization/convrot_int8_comfy_kitchen.py +++ b/python/sglang/multimodal_gen/runtime/layers/quantization/convrot_int8_comfy_kitchen.py @@ -30,7 +30,7 @@ from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_conf ) from sglang.multimodal_gen.runtime.utils.weight_attrs import set_weight_attrs -__all__ = ["ConvRotInt8ComfyKitchenLinearMethod"] +__all__ = ["ConvRotInt8ComfyKitchenLinearMethod", "comfy_kitchen_swiglu_mlp"] # comfy_kitchen's dtype codes for the fused op's output. _OUT_DTYPE_CODE = {torch.float32: 0, torch.float16: 1, torch.bfloat16: 2} @@ -221,3 +221,37 @@ class ConvRotInt8ComfyKitchenLinearMethod(LinearMethodBase): if len(orig_shape) != 2: out = out.reshape(*orig_shape[:-1], out.shape[-1]) return out + + +def comfy_kitchen_swiglu_mlp( + x: torch.Tensor, + gate_up_weight: torch.Tensor, + gate_up_scale: torch.Tensor, + gate_up_group_size: int, + down: torch.nn.Module, +) -> torch.Tensor: + """``down(silu(gate) * up)`` for ``[gate | up] = x @ gate_up_weight.T``, run as + ComfyUI runs SwiGLU MLPs: one GEMM over the stacked [gate; up] weight, and a + down-projection that applies the SwiGLU inside its quantizer + (``input_act="swiglu"``), so the activated rows never reach HBM. + + The fused quantizer keeps ``silu(gate) * up`` in FP32 instead of rounding it + to BF16 first, so the result is not bitwise the eager form; it sits slightly + closer to an FP32 reference. + """ + out_code = _OUT_DTYPE_CODE[x.dtype] + rows = x.reshape(-1, x.shape[-1]).contiguous() + gate_up = torch.ops.comfy_kitchen.int8_linear( + rows, gate_up_weight, gate_up_scale, None, out_code, True, gate_up_group_size + ) + out = torch.ops.comfy_kitchen.int8_linear( + gate_up, + down.weight, + down.weight_scale, + None, + out_code, + True, # convrot + down.quant_method.group_size, + "swiglu", + ) + return out.reshape(*x.shape[:-1], out.shape[-1]) diff --git a/python/sglang/multimodal_gen/runtime/models/dits/qwen_image21.py b/python/sglang/multimodal_gen/runtime/models/dits/qwen_image21.py index a8f62de631..d8c1a9b4f0 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/qwen_image21.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/qwen_image21.py @@ -45,6 +45,10 @@ from sglang.multimodal_gen.runtime.layers.linear import ( UnquantizedLinearMethod, ) from sglang.multimodal_gen.runtime.layers.lora.linear import BaseLayerWithLoRA +from sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_comfy_kitchen import ( + ConvRotInt8ComfyKitchenLinearMethod, + comfy_kitchen_swiglu_mlp, +) from sglang.multimodal_gen.runtime.managers.forward_context import get_forward_context from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import ( LayerwiseOffloadableModuleMixin, @@ -318,6 +322,22 @@ class QwenImage21TimeEmbedding(nn.Module): ) +def _stacked_rows(a, b): + """[a; b] as one view when ``b`` directly follows ``a`` in the same storage, else None.""" + # Layerwise offload leaves a flat placeholder in place of each weight. + if not ( + a.dim() == 2 + and a.is_cuda + and a.shape == b.shape + and a.is_contiguous() + and b.is_contiguous() + and a.untyped_storage().data_ptr() == b.untyped_storage().data_ptr() + and b.storage_offset() == a.storage_offset() + a.numel() + ): + return None + return a.as_strided((2 * a.shape[0], a.shape[1]), (a.shape[1], 1)) + + class QwenImage21FeedForward(nn.Module): def __init__(self, dim, ratio, quant_config, prefix): super().__init__() @@ -343,7 +363,72 @@ class QwenImage21FeedForward(nn.Module): prefix=f"{prefix}.out", ) + def pack_gate_up_weights(self): + """Share one [2F, C] INT8 weight and [2F, 1] scale storage between gate_layer + and proj so the comfy_kitchen path projects both with one GEMM. + + As with ``pack_qkv_weights``, parameters keep their names and shapes and + no reference to the shared storage is kept. Layers that are not plain + comfy_kitchen INT8 projections are left alone. + """ + gate, up = self.gate_layer, self.proj + if self.packed_gate_up() is not None: + return + if not ( + all( + type(layer) is ColumnParallelLinear + and isinstance(layer.quant_method, ConvRotInt8ComfyKitchenLinearMethod) + and layer.weight.is_cuda + and layer.weight.dtype is torch.int8 + and layer.weight.dim() == 2 + for layer in (gate, up) + ) + and gate.weight.shape == up.weight.shape + and gate.quant_method.group_size == up.quant_method.group_size + ): + return + rows = gate.weight.shape[0] + weight = torch.cat([gate.weight.data, up.weight.data], dim=0) + scale = torch.cat([gate.weight_scale.data, up.weight_scale.data], dim=0) + gate.weight.data, up.weight.data = weight[:rows], weight[rows:] + gate.weight_scale.data, up.weight_scale.data = scale[:rows], scale[rows:] + + def packed_gate_up(self): + """The stacked [gate; up] weight and scale views when the comfy_kitchen + fused MLP can run, else None.""" + gate, up, out = self.gate_layer, self.proj, self.out + # LoRA wrappers replace the layers; TP > 1 would need the down all-reduce. + if not ( + type(gate) is ColumnParallelLinear + and type(up) is ColumnParallelLinear + and type(out) is RowParallelLinear + and out.tp_size == 1 + and isinstance(gate.quant_method, ConvRotInt8ComfyKitchenLinearMethod) + and isinstance(out.quant_method, ConvRotInt8ComfyKitchenLinearMethod) + and out.weight.dtype is torch.int8 + ): + return None + weight = _stacked_rows(gate.weight, up.weight) + scale = _stacked_rows(gate.weight_scale, up.weight_scale) + if weight is None or scale is None: + return None + return weight, scale + + def _apply(self, fn, recurse=True): + # .to() and CPU offload rebuild every parameter in its own storage; + # re-share them once the weights are back on the GPU. + module = super()._apply(fn, recurse) + self.pack_gate_up_weights() + return module + def forward(self, x): + # Storage-identity checks cannot be traced; a compiled graph keeps the + # separate projections. + packed = None if torch.compiler.is_compiling() else self.packed_gate_up() + if packed is not None: + return comfy_kitchen_swiglu_mlp( + x, *packed, self.gate_layer.quant_method.group_size, self.out + ) gate, value = self.gate_layer(x)[0], self.proj(x)[0] fused = None if ( @@ -917,6 +1002,7 @@ class QwenImage21Transformer2DModel(CachableDiT, LayerwiseOffloadableModuleMixin super().post_load_weights() for block in self.transformer_blocks: block.attn.pack_qkv_weights() + block.img_mlp.pack_gate_up_weights() def prepare_lora_adapter(self, adapter): return _split_fused_gate_up_lora( diff --git a/python/sglang/multimodal_gen/test/unit/test_convrot_int8_backends.py b/python/sglang/multimodal_gen/test/unit/test_convrot_int8_backends.py index 9c52ee18ba..49b85dd321 100644 --- a/python/sglang/multimodal_gen/test/unit/test_convrot_int8_backends.py +++ b/python/sglang/multimodal_gen/test/unit/test_convrot_int8_backends.py @@ -19,6 +19,7 @@ from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_conf ) from sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_comfy_kitchen import ( ConvRotInt8ComfyKitchenLinearMethod, + comfy_kitchen_available, ) from sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_jit import ( ConvRotInt8JitLinearMethod, @@ -219,6 +220,57 @@ def test_gelu_input_helper_is_bitwise_eager_gelu_then_layer(): assert torch.equal(fused, eager) +@pytest.mark.skipif( + not (torch.cuda.is_available() and comfy_kitchen_available()), + reason="needs CUDA and comfy_kitchen", +) +def test_qwen_image21_kitchen_mlp_packs_gate_up_and_folds_swiglu(monkeypatch): + """On comfy_kitchen the Qwen-Image 2.1 MLP runs ComfyUI's fused form; once + the shared gate/up storage is gone it falls back to separate projections.""" + from sglang.multimodal_gen.runtime.models.dits.qwen_image21 import ( + QwenImage21FeedForward, + ) + + _with_tp(monkeypatch, 1) + ffn = QwenImage21FeedForward( + 1024, 3, ConvRotInt8Config(backend="comfy_kitchen"), "blocks.0.img_mlp" + ).cuda() + gen = torch.Generator(device="cuda").manual_seed(0) + for layer in (ffn.gate_layer, ffn.proj, ffn.out): + with torch.no_grad(): + layer.weight.copy_( + torch.randn(layer.weight.shape, device="cuda", generator=gen) * 0.02 + ) + out_weight = layer.weight.detach().float() # ends as ffn.out's BF16 weight + layer.quant_method.process_weights_after_loading(layer) + x = torch.randn(1, 2048, 1024, device="cuda", dtype=torch.bfloat16, generator=gen) + gate, up = ffn.gate_layer(x)[0], ffn.proj(x)[0] + eager = ffn.out(F.silu(gate) * up)[0] + + ffn.pack_gate_up_weights() + assert ffn.packed_gate_up() is not None + assert ffn.gate_layer.weight.shape == ffn.proj.weight.shape == (3072, 1024) + fused = ffn(x) + # The fused quantizer keeps silu(gate) * up in FP32 rather than rounding it + # to BF16, so it is not bitwise the eager form; it must be no further (1% + # slack) from the FP32 down-projection of the same gate and up. + reference = (F.silu(gate.float()) * up.float()) @ out_weight.t() + + def rel_l2(out): + err = torch.linalg.vector_norm(out.float() - reference) + return (err / torch.linalg.vector_norm(reference)).item() + + assert rel_l2(fused) <= 1.01 * rel_l2(eager), (rel_l2(fused), rel_l2(eager)) + + # A CPU round trip rebuilds every parameter on its own; .cuda() re-packs. + ffn.cpu().cuda() + assert torch.equal(ffn(x), fused) + + ffn.proj.weight.data = ffn.proj.weight.data.clone() + assert ffn.packed_gate_up() is None + assert torch.equal(ffn(x), eager) + + class _FakeTpGroup: pass