[diffusion] feat: support loading community ComfyUI-GGUF Qwen-Image-2.1 transformers (#42556)
Co-authored-by: Mick <mickjagger19@icloud.com>
This commit is contained in:
@@ -35,6 +35,7 @@ from sglang.multimodal_gen.runtime.layers.quantization.configs.nunchaku_config i
|
||||
from sglang.multimodal_gen.runtime.loader.gguf_weights import (
|
||||
names_gguf_checkpoint,
|
||||
read_gguf_tensor_meta,
|
||||
remap_gguf_tensor_meta,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.loader.utils import _list_safetensors_files
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.component_residency import (
|
||||
@@ -791,10 +792,13 @@ def _resolve_gguf_quant_load_spec(
|
||||
|
||||
_validate_gguf_runtime_support(server_args, component_name)
|
||||
|
||||
quant_config = GGUFConfig(
|
||||
gguf_file=gguf_file,
|
||||
tensor_meta=read_gguf_tensor_meta(gguf_file),
|
||||
)
|
||||
tensor_meta = read_gguf_tensor_meta(gguf_file)
|
||||
dequantize_prefixes = vars(model_cls).get("gguf_dequantize_prefixes", ())
|
||||
if dequantize_prefixes:
|
||||
tensor_meta = remap_gguf_tensor_meta(
|
||||
tensor_meta, lambda name: name, dequantize_prefixes=dequantize_prefixes
|
||||
)
|
||||
quant_config = GGUFConfig(gguf_file=gguf_file, tensor_meta=tensor_meta)
|
||||
packed = getattr(model_cls, "packed_modules_mapping", None)
|
||||
if packed:
|
||||
quant_config.packed_modules_mapping = packed
|
||||
|
||||
@@ -41,6 +41,7 @@ from sglang.multimodal_gen.runtime.layers.linear import (
|
||||
RowParallelLinear,
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.lora.linear import BaseLayerWithLoRA
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
|
||||
LayerwiseOffloadableModuleMixin,
|
||||
)
|
||||
@@ -399,6 +400,17 @@ class QwenImage21Attention(nn.Module):
|
||||
|
||||
def packed_qkv_weight(self):
|
||||
"""The [3C, C] view over to_q/to_k/to_v when they are consecutive slices of one storage, else None."""
|
||||
# LoRA wrappers keep the projection, and its quant method, in base_layer.
|
||||
if not all(
|
||||
isinstance(
|
||||
(
|
||||
layer.base_layer if isinstance(layer, BaseLayerWithLoRA) else layer
|
||||
).quant_method,
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
for layer in (self.to_q, self.to_k, self.to_v)
|
||||
):
|
||||
return None
|
||||
q, k, v = self.to_q.weight, self.to_k.weight, self.to_v.weight
|
||||
rows, cols = q.shape
|
||||
if not (
|
||||
@@ -807,6 +819,15 @@ class QwenImage21Transformer2DModel(CachableDiT, LayerwiseOffloadableModuleMixin
|
||||
_compile_conditions = _fsdp_shard_conditions
|
||||
layer_names = ["transformer_blocks"]
|
||||
param_names_mapping = {}
|
||||
# Plain nn.Linear outside the blocks: GGUF quantizes them, so dequantize on load.
|
||||
gguf_dequantize_prefixes = (
|
||||
"img_in.",
|
||||
"txt_in.",
|
||||
"time_text_embed.",
|
||||
"modulation.",
|
||||
"norm_out.",
|
||||
"proj_out.",
|
||||
)
|
||||
|
||||
def __init__(self, config, hf_config, quant_config=None, **kwargs):
|
||||
super().__init__(config, hf_config=hf_config, **kwargs)
|
||||
|
||||
@@ -17,6 +17,10 @@ import numpy as np
|
||||
import torch
|
||||
from gguf import GGMLQuantizationType as WeightType
|
||||
|
||||
from sglang.multimodal_gen.configs.models.dits.qwenimage21 import (
|
||||
QwenImage21ArchConfig,
|
||||
QwenImage21DitConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
MergedColumnParallelLinear,
|
||||
@@ -35,6 +39,13 @@ from sglang.multimodal_gen.runtime.loader.gguf_weights import (
|
||||
read_gguf_tensor_meta,
|
||||
remap_gguf_tensor_meta,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.loader.transformer_load_utils import (
|
||||
_resolve_gguf_quant_load_spec,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.dits.qwen_image21 import (
|
||||
QwenImage21Attention,
|
||||
QwenImage21Transformer2DModel,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.encoders.minimax_h3_qwen3vl import (
|
||||
MiniMaxH3Qwen3VLEncoder,
|
||||
)
|
||||
@@ -950,5 +961,73 @@ class TestGGUFRejectsLoraConversion(unittest.TestCase):
|
||||
entered.assert_not_called()
|
||||
|
||||
|
||||
class TestQwenImage21GGUF(unittest.TestCase):
|
||||
"""Community Qwen-Image 2.1 GGUFs also quantize the DiT's plain nn.Linear layers."""
|
||||
|
||||
def test_plain_linears_dequantize_while_block_linears_stay_packed(self):
|
||||
"""A packed `modulation.1.qweight` has no nn.Linear to land in and failed the load."""
|
||||
payload = bytes(4 * 512 // _Q4_K_BLOCK * _Q4_K_TYPE_SIZE)
|
||||
server_args = Mock(
|
||||
use_fsdp_inference=False,
|
||||
lora_path=None,
|
||||
minimax_h3_adaln_online=False,
|
||||
minimax_h3_adaln_cache_path=None,
|
||||
quantization=None,
|
||||
nunchaku_config=None,
|
||||
)
|
||||
with (
|
||||
tempfile.TemporaryDirectory() as tmp,
|
||||
patch(
|
||||
"sglang.multimodal_gen.runtime.loader.transformer_load_utils.current_platform"
|
||||
) as platform,
|
||||
):
|
||||
path = str(Path(tmp) / "qwen21.gguf")
|
||||
_write_gguf(
|
||||
Path(path),
|
||||
[
|
||||
("modulation.1.weight", [512, 4], _Q4_K, payload),
|
||||
("transformer_blocks.0.attn.to_q.weight", [512, 4], _Q4_K, payload),
|
||||
],
|
||||
)
|
||||
platform.is_cuda.return_value = True
|
||||
spec = _resolve_gguf_quant_load_spec(
|
||||
gguf_file=path,
|
||||
server_args=server_args,
|
||||
model_cls=QwenImage21Transformer2DModel,
|
||||
)
|
||||
loaded = dict(gguf_weights_iterator(path, spec.quant_config.tensor_meta))
|
||||
|
||||
self.assertEqual(loaded["modulation.1.weight"].dtype, torch.bfloat16)
|
||||
self.assertIn("transformer_blocks.0.attn.to_q.qweight", loaded)
|
||||
|
||||
def test_every_plain_linear_is_covered_by_dequantize_prefixes(self):
|
||||
"""A new nn.Linear outside the blocks must also join gguf_dequantize_prefixes."""
|
||||
config = QwenImage21DitConfig(arch_config=QwenImage21ArchConfig(num_layers=0))
|
||||
with torch.device("meta"):
|
||||
model = QwenImage21Transformer2DModel(config, {})
|
||||
prefixes = QwenImage21Transformer2DModel.gguf_dequantize_prefixes
|
||||
|
||||
for name, module in model.named_modules():
|
||||
if type(module) is torch.nn.Linear:
|
||||
self.assertTrue(f"{name}.weight".startswith(prefixes), name)
|
||||
|
||||
def test_qkv_packing_skips_gguf_projections(self):
|
||||
"""GGUF projections register only `qweight`; reading `.weight` crashed post-load."""
|
||||
meta = GGUFTensorMeta(
|
||||
ggml_type=int(_Q4_K),
|
||||
logical_shape=(4, 512),
|
||||
stored_shape=(4, 288),
|
||||
stored_dtype=torch.uint8,
|
||||
param_name="w.qweight",
|
||||
)
|
||||
config = GGUFConfig("/dev/null", {"w.weight": meta})
|
||||
projection = ReplicatedLinear(
|
||||
512, 4, bias=False, quant_config=config, prefix="w"
|
||||
)
|
||||
attention = SimpleNamespace(to_q=projection, to_k=projection, to_v=projection)
|
||||
|
||||
self.assertIsNone(QwenImage21Attention.packed_qkv_weight(attention))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user