diff --git a/docs/cookbook/diffusion/Qwen-Image/Qwen-Image-2.1.mdx b/docs/cookbook/diffusion/Qwen-Image/Qwen-Image-2.1.mdx index 6cf0a37bf0..302e853717 100644 --- a/docs/cookbook/diffusion/Qwen-Image/Qwen-Image-2.1.mdx +++ b/docs/cookbook/diffusion/Qwen-Image/Qwen-Image-2.1.mdx @@ -214,8 +214,10 @@ features below. Functional checks do not establish quality for lossy settings. multi-rank roles remain unverified for this model. - **Quantized exports:** serialized FP8, native-name Q4_0 GGUF, and calibrated ModelOpt NVFP4 component exports were checked on B200, including TP2 with - encoder folding and separate offload checks. These results do not establish - support for arbitrary community exports, other GGUF types, or other hardware. + encoder folding and separate offload checks. A community ComfyUI-GGUF Q4_K_M + DiT, which also quantizes the projections outside the blocks, was checked for + text-to-image on one RTX 5090. These results do not establish support for + arbitrary community exports, other GGUF types, or other hardware. The NVFP4 export used six calibration requests; assess image and alpha quality on your workload. - **LoRA:** synthetic Diffusers-format adapters were checked on one B200 and diff --git a/docs/docs/sglang-diffusion/quantization.mdx b/docs/docs/sglang-diffusion/quantization.mdx index 227b6d0797..bfc60e1749 100644 --- a/docs/docs/sglang-diffusion/quantization.mdx +++ b/docs/docs/sglang-diffusion/quantization.mdx @@ -987,6 +987,7 @@ startup rejects a degree that cuts a row-parallel matrix inside a GGML block. | MiniMax-H3 `fl2va`, pruned AdaLN curve | `minimax_h3_fl2va_pruned-Q4_K.gguf` | 1x GB300 (CUDA 13, PyTorch 2.13) | 50-step t2va 1344x768, 107 frames, H.264 + AAC; 105.38 s and 80.88 GB peak | | MiniMax-H3 `fl2va`, pruned AdaLN curve | `minimax_h3_fl2va_pruned-Q4_K.gguf` | 2x GB300, TP2 (CUDA 13, PyTorch 2.13) | 2-step t2va 1344x768, 107 frames, H.264 + AAC; 7.55 s and 51.90 GB peak per rank | | Qwen-Image 2.1 | Private native-name Q4_0 exports; DiT 3.91 GiB, encoder 7.03 GiB | 1x B200; separate TP2 check | 1024px/40-step generation, editing, and transparent RGBA; each component and both together; combined offload matched resident pixels | +| Qwen-Image 2.1 | Community ComfyUI-GGUF DiT [`abenzerps/Qwen-Image-2.1-Uncensored-GGUF`](https://huggingface.co/abenzerps/Qwen-Image-2.1-Uncensored-GGUF) `qwen-image-2.1-UC-Q4_K_M.gguf` (4.6 GB) | 1x RTX 5090 (32 GiB) | 1024px/40-step text-to-image; 8.25 GB peak vs 17.0 GB for the BF16 DiT, 0.47 vs 0.43-0.48 s/step | The unpruned MiniMax-H3 DiT loads at 17.5 GiB against 61.7 GiB for the BF16 checkpoint. Weight fidelity was checked tensor-by-tensor against the BF16 reference: cosine diff --git a/python/sglang/multimodal_gen/runtime/loader/transformer_load_utils.py b/python/sglang/multimodal_gen/runtime/loader/transformer_load_utils.py index 5a650e45ce..54ed9c033f 100644 --- a/python/sglang/multimodal_gen/runtime/loader/transformer_load_utils.py +++ b/python/sglang/multimodal_gen/runtime/loader/transformer_load_utils.py @@ -35,6 +35,7 @@ from sglang.multimodal_gen.runtime.layers.quantization.configs.nunchaku_config i from sglang.multimodal_gen.runtime.loader.gguf_weights import ( names_gguf_checkpoint, read_gguf_tensor_meta, + remap_gguf_tensor_meta, ) from sglang.multimodal_gen.runtime.loader.utils import _list_safetensors_files from sglang.multimodal_gen.runtime.managers.memory_managers.component_residency import ( @@ -791,10 +792,13 @@ def _resolve_gguf_quant_load_spec( _validate_gguf_runtime_support(server_args, component_name) - quant_config = GGUFConfig( - gguf_file=gguf_file, - tensor_meta=read_gguf_tensor_meta(gguf_file), - ) + tensor_meta = read_gguf_tensor_meta(gguf_file) + dequantize_prefixes = vars(model_cls).get("gguf_dequantize_prefixes", ()) + if dequantize_prefixes: + tensor_meta = remap_gguf_tensor_meta( + tensor_meta, lambda name: name, dequantize_prefixes=dequantize_prefixes + ) + quant_config = GGUFConfig(gguf_file=gguf_file, tensor_meta=tensor_meta) packed = getattr(model_cls, "packed_modules_mapping", None) if packed: quant_config.packed_modules_mapping = packed diff --git a/python/sglang/multimodal_gen/runtime/models/dits/qwen_image21.py b/python/sglang/multimodal_gen/runtime/models/dits/qwen_image21.py index 564fce4eb8..f593d33d84 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/qwen_image21.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/qwen_image21.py @@ -41,6 +41,7 @@ from sglang.multimodal_gen.runtime.layers.linear import ( RowParallelLinear, UnquantizedLinearMethod, ) +from sglang.multimodal_gen.runtime.layers.lora.linear import BaseLayerWithLoRA from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import ( LayerwiseOffloadableModuleMixin, ) @@ -399,6 +400,17 @@ class QwenImage21Attention(nn.Module): def packed_qkv_weight(self): """The [3C, C] view over to_q/to_k/to_v when they are consecutive slices of one storage, else None.""" + # LoRA wrappers keep the projection, and its quant method, in base_layer. + if not all( + isinstance( + ( + layer.base_layer if isinstance(layer, BaseLayerWithLoRA) else layer + ).quant_method, + UnquantizedLinearMethod, + ) + for layer in (self.to_q, self.to_k, self.to_v) + ): + return None q, k, v = self.to_q.weight, self.to_k.weight, self.to_v.weight rows, cols = q.shape if not ( @@ -807,6 +819,15 @@ class QwenImage21Transformer2DModel(CachableDiT, LayerwiseOffloadableModuleMixin _compile_conditions = _fsdp_shard_conditions layer_names = ["transformer_blocks"] param_names_mapping = {} + # Plain nn.Linear outside the blocks: GGUF quantizes them, so dequantize on load. + gguf_dequantize_prefixes = ( + "img_in.", + "txt_in.", + "time_text_embed.", + "modulation.", + "norm_out.", + "proj_out.", + ) def __init__(self, config, hf_config, quant_config=None, **kwargs): super().__init__(config, hf_config=hf_config, **kwargs) diff --git a/python/sglang/multimodal_gen/test/unit/test_gguf_diffusion.py b/python/sglang/multimodal_gen/test/unit/test_gguf_diffusion.py index c20748fc6a..eee28b1aed 100644 --- a/python/sglang/multimodal_gen/test/unit/test_gguf_diffusion.py +++ b/python/sglang/multimodal_gen/test/unit/test_gguf_diffusion.py @@ -17,6 +17,10 @@ import numpy as np import torch from gguf import GGMLQuantizationType as WeightType +from sglang.multimodal_gen.configs.models.dits.qwenimage21 import ( + QwenImage21ArchConfig, + QwenImage21DitConfig, +) from sglang.multimodal_gen.runtime.layers.linear import ( ColumnParallelLinear, MergedColumnParallelLinear, @@ -35,6 +39,13 @@ from sglang.multimodal_gen.runtime.loader.gguf_weights import ( read_gguf_tensor_meta, remap_gguf_tensor_meta, ) +from sglang.multimodal_gen.runtime.loader.transformer_load_utils import ( + _resolve_gguf_quant_load_spec, +) +from sglang.multimodal_gen.runtime.models.dits.qwen_image21 import ( + QwenImage21Attention, + QwenImage21Transformer2DModel, +) from sglang.multimodal_gen.runtime.models.encoders.minimax_h3_qwen3vl import ( MiniMaxH3Qwen3VLEncoder, ) @@ -950,5 +961,73 @@ class TestGGUFRejectsLoraConversion(unittest.TestCase): entered.assert_not_called() +class TestQwenImage21GGUF(unittest.TestCase): + """Community Qwen-Image 2.1 GGUFs also quantize the DiT's plain nn.Linear layers.""" + + def test_plain_linears_dequantize_while_block_linears_stay_packed(self): + """A packed `modulation.1.qweight` has no nn.Linear to land in and failed the load.""" + payload = bytes(4 * 512 // _Q4_K_BLOCK * _Q4_K_TYPE_SIZE) + server_args = Mock( + use_fsdp_inference=False, + lora_path=None, + minimax_h3_adaln_online=False, + minimax_h3_adaln_cache_path=None, + quantization=None, + nunchaku_config=None, + ) + with ( + tempfile.TemporaryDirectory() as tmp, + patch( + "sglang.multimodal_gen.runtime.loader.transformer_load_utils.current_platform" + ) as platform, + ): + path = str(Path(tmp) / "qwen21.gguf") + _write_gguf( + Path(path), + [ + ("modulation.1.weight", [512, 4], _Q4_K, payload), + ("transformer_blocks.0.attn.to_q.weight", [512, 4], _Q4_K, payload), + ], + ) + platform.is_cuda.return_value = True + spec = _resolve_gguf_quant_load_spec( + gguf_file=path, + server_args=server_args, + model_cls=QwenImage21Transformer2DModel, + ) + loaded = dict(gguf_weights_iterator(path, spec.quant_config.tensor_meta)) + + self.assertEqual(loaded["modulation.1.weight"].dtype, torch.bfloat16) + self.assertIn("transformer_blocks.0.attn.to_q.qweight", loaded) + + def test_every_plain_linear_is_covered_by_dequantize_prefixes(self): + """A new nn.Linear outside the blocks must also join gguf_dequantize_prefixes.""" + config = QwenImage21DitConfig(arch_config=QwenImage21ArchConfig(num_layers=0)) + with torch.device("meta"): + model = QwenImage21Transformer2DModel(config, {}) + prefixes = QwenImage21Transformer2DModel.gguf_dequantize_prefixes + + for name, module in model.named_modules(): + if type(module) is torch.nn.Linear: + self.assertTrue(f"{name}.weight".startswith(prefixes), name) + + def test_qkv_packing_skips_gguf_projections(self): + """GGUF projections register only `qweight`; reading `.weight` crashed post-load.""" + meta = GGUFTensorMeta( + ggml_type=int(_Q4_K), + logical_shape=(4, 512), + stored_shape=(4, 288), + stored_dtype=torch.uint8, + param_name="w.qweight", + ) + config = GGUFConfig("/dev/null", {"w.weight": meta}) + projection = ReplicatedLinear( + 512, 4, bias=False, quant_config=config, prefix="w" + ) + attention = SimpleNamespace(to_q=projection, to_k=projection, to_v=projection) + + self.assertIsNone(QwenImage21Attention.packed_qkv_weight(attention)) + + if __name__ == "__main__": unittest.main()