[Feat] ComfyUI: Add online INT8 quantization for BF16 Qwen-Image-2.1 single files (#44865)
Co-authored-by: WenhaoZhang <42087078+niehen6174@users.noreply.github.com>
This commit is contained in:
co-authored by
WenhaoZhang
parent
a361f0c410
commit
fee32a5c1f
@@ -59,7 +59,7 @@ extension, which only changes the presentation.
|
||||
| --- | --- |
|
||||
| Parallel | `parallel_mode` auto or manual; manual sets `tp_size`, `ulysses_degree`, `ring_degree`, whose product must equal `num_gpus` |
|
||||
| Attention | `attention_backend`: auto (model default), flash_attention, torch_sdpa, sage_attention2, sage_attention3, sol_attention (H3; needs the `sol-attn` package; picking it fills `attention_backend_config` with its defaults), and the H3 sparse backends; `attention_backend_config` |
|
||||
| Quantization | `quantization` (online, native directories only); `int8_gemm`: GEMM implementation for INT8 ConvRot weights, auto / sglang_jit / comfy_kitchen |
|
||||
| Quantization | `quantization` (online: native directories, or `convrot_int8` on a BF16 Qwen-Image-2.1 file); `int8_gemm`: GEMM implementation for INT8 ConvRot weights, auto / sglang_jit / comfy_kitchen |
|
||||
| Cache | `cache_mode`: off, cache_dit, teacache, spectrum (lossy). For cache_dit, `cache_dit_tuning` auto keeps the defaults; manual sets warmup steps, residual threshold, max consecutive cached steps and Fn compute blocks |
|
||||
| Execution | `execution_mode`: eager, torch_compile, cuda_graph (breakable CUDA graph); `regional_compile` |
|
||||
| Memory | `memory_mode`: resident or DiT layerwise offload; `resident_layers`, `prefetch_layers` |
|
||||
@@ -72,7 +72,7 @@ worker and raises instead of dropping a setting. Current limits in integrated mo
|
||||
| cuda_graph | MiniMax H3, with `cache_mode` off and no layerwise offload |
|
||||
| torch_compile | all but FastH3 and VDN-H3 |
|
||||
| ring_degree > 1 | not FastH3 / VDN-H3; MiniMax H3 needs `fa` |
|
||||
| quantization | native model directories; ComfyUI files carry their own format |
|
||||
| quantization | native model directories; BF16 Qwen-Image-2.1 files with `convrot_int8`. Other ComfyUI files carry their own format |
|
||||
| cache_dit, spectrum | MiniMax H3 family (spectrum: sequence-parallel degree 1); other models send each sampler step as its own request, so a step cache needs per-run state first |
|
||||
| teacache | none yet |
|
||||
|
||||
|
||||
@@ -133,6 +133,12 @@ _ATTENTION_BY_MODEL = {
|
||||
}
|
||||
|
||||
|
||||
# Model types whose BF16 ComfyUI file has been validated with online quantization.
|
||||
_SINGLE_FILE_ONLINE_QUANT = frozenset({"qwen_image21"})
|
||||
# Online FP8 drifts on Qwen-Image 2.1 (SSIM ~0.85 vs BF16), so only INT8 is allowed.
|
||||
_SINGLE_FILE_ONLINE_METHODS = frozenset({"convrot_int8"})
|
||||
|
||||
|
||||
def _unsupported(feature: str, model_type: str) -> ValueError:
|
||||
return ValueError(
|
||||
f"{feature} is not supported for {model_type} in ComfyUI integrated mode yet"
|
||||
@@ -199,14 +205,25 @@ def _check_cache(model_type: str, opts: dict) -> None:
|
||||
|
||||
|
||||
def _check_weights(model_type: str, opts: dict, *, native_dir: bool) -> None:
|
||||
if opts.get("quantization") and not native_dir:
|
||||
raise ValueError(
|
||||
"quantization (online, at load) applies to native model directories; "
|
||||
"ComfyUI files carry their own format, so load a pre-quantized file "
|
||||
"(e.g. *_int8_convrot) instead"
|
||||
)
|
||||
quantization = opts.get("quantization")
|
||||
if quantization and not native_dir:
|
||||
if model_type not in _SINGLE_FILE_ONLINE_QUANT:
|
||||
raise ValueError(
|
||||
f"quantization (online, at load) of a ComfyUI file is supported for "
|
||||
f"{sorted(_SINGLE_FILE_ONLINE_QUANT)}; for {model_type} use a native "
|
||||
"model directory or a pre-quantized file (e.g. *_int8_convrot)"
|
||||
)
|
||||
if quantization not in _SINGLE_FILE_ONLINE_METHODS:
|
||||
raise ValueError(
|
||||
f"quantization={quantization} needs a pre-quantized checkpoint; a "
|
||||
f"BF16 ComfyUI file takes one of {sorted(_SINGLE_FILE_ONLINE_METHODS)}"
|
||||
)
|
||||
kernel = opts.get("int8_kernel", "auto")
|
||||
if kernel != "auto" and model_type not in H3_MODELS:
|
||||
if (
|
||||
kernel != "auto"
|
||||
and model_type not in H3_MODELS
|
||||
and model_type not in {"qwen_image21"}
|
||||
):
|
||||
raise ValueError(
|
||||
f"int8_gemm={kernel} has no effect on {model_type}: only MiniMax H3 "
|
||||
"runs ConvRot INT8 weights; leave it at auto"
|
||||
|
||||
@@ -55,7 +55,8 @@ _FIELDS = {
|
||||
"quantization",
|
||||
["none", "fp8", "convrot_int8", "mxfp8"],
|
||||
"none",
|
||||
"Online quantization; native model directories only.",
|
||||
"Online quantization: native model directories, or a BF16 Qwen-Image 2.1 "
|
||||
"file (convrot_int8).",
|
||||
),
|
||||
"int8_gemm": _field(
|
||||
"quantization",
|
||||
|
||||
@@ -77,6 +77,9 @@ class ComfyUICheckpointSpec:
|
||||
checkpoint_key_filter: Callable[[str], bool] | None = None
|
||||
|
||||
|
||||
# Methods whose no-arg config quantizes BF16 weights after loading.
|
||||
_ONLINE_QUANTIZATIONS = frozenset({"fp8", "mxfp4", "convrot_int8"})
|
||||
|
||||
_SPEC_REGISTRY: dict[str, ComfyUICheckpointSpec] = {}
|
||||
_SPECS_DISCOVERED = False
|
||||
|
||||
@@ -191,6 +194,51 @@ def _load_comfyui_gguf_transformer(
|
||||
model_cls.param_names_mapping = original_mapping
|
||||
|
||||
|
||||
def _resolve_online_quant_spec(
|
||||
*,
|
||||
server_args: ServerArgs,
|
||||
model_cls,
|
||||
cls_name: str,
|
||||
model_path: str,
|
||||
dit_config,
|
||||
):
|
||||
from sglang.multimodal_gen.runtime.loader.transformer_load_utils import (
|
||||
resolve_transformer_quant_load_spec,
|
||||
)
|
||||
|
||||
if server_args.quantization not in _ONLINE_QUANTIZATIONS:
|
||||
raise ValueError(
|
||||
f"--quantization {server_args.quantization} needs a pre-quantized "
|
||||
"checkpoint; a BF16 ComfyUI file can only be quantized online with "
|
||||
f"one of {sorted(_ONLINE_QUANTIZATIONS)}"
|
||||
)
|
||||
return resolve_transformer_quant_load_spec(
|
||||
hf_config={},
|
||||
server_args=server_args,
|
||||
safetensors_list=[model_path],
|
||||
component_model_path=os.path.dirname(model_path) or ".",
|
||||
model_cls=model_cls,
|
||||
cls_name=cls_name,
|
||||
component_name="transformer",
|
||||
arch_config=dit_config.arch_config,
|
||||
)
|
||||
|
||||
|
||||
def _online_quant_load_plan(*, server_args: ServerArgs, quant_spec) -> WeightLoadPlan:
|
||||
starts_on_cpu = server_args.should_start_component_on_cpu("transformer")
|
||||
# Matches the native loader: only ConvRot INT8 quantizes from host weights.
|
||||
checkpoint_load_device = (
|
||||
torch.device("cpu")
|
||||
if starts_on_cpu and quant_spec.is_convrot_int8
|
||||
else get_local_torch_device()
|
||||
)
|
||||
return WeightLoadPlan.for_component(
|
||||
checkpoint_load_device=checkpoint_load_device,
|
||||
needs_device_weight_postprocess=quant_spec.needs_device_weight_postprocess,
|
||||
component_starts_on_cpu=starts_on_cpu,
|
||||
)
|
||||
|
||||
|
||||
def load_comfyui_transformer(
|
||||
pipeline: ComposedPipelineBase,
|
||||
server_args: ServerArgs,
|
||||
@@ -312,6 +360,23 @@ def load_comfyui_transformer(
|
||||
adaln_curve_shape,
|
||||
)
|
||||
|
||||
post_load_hooks = []
|
||||
if quant_config is None and server_args.quantization is not None:
|
||||
# A BF16 file has no markers; quantize it online like a native directory.
|
||||
quant_spec = _resolve_online_quant_spec(
|
||||
server_args=server_args,
|
||||
model_cls=model_cls,
|
||||
cls_name=spec.dit_cls_name,
|
||||
model_path=model_path,
|
||||
dit_config=dit_config,
|
||||
)
|
||||
quant_config = quant_spec.quant_config
|
||||
param_dtype = quant_spec.param_dtype
|
||||
post_load_hooks = quant_spec.post_load_hooks
|
||||
weight_load_plan = _online_quant_load_plan(
|
||||
server_args=server_args, quant_spec=quant_spec
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Loading %s from ComfyUI checkpoint %s, param_dtype: %s",
|
||||
spec.dit_cls_name,
|
||||
@@ -349,6 +414,8 @@ def load_comfyui_transformer(
|
||||
)
|
||||
finally:
|
||||
model_cls.param_names_mapping = original_mapping
|
||||
for post_load_hook in post_load_hooks:
|
||||
post_load_hook(model)
|
||||
|
||||
for param in model.parameters():
|
||||
param.requires_grad = False
|
||||
|
||||
@@ -10,7 +10,7 @@ from sglang.multimodal_gen.runtime.loader.comfyui_checkpoints import spec as spe
|
||||
_MARKER = {"format": "int8_tensorwise", "convrot": True, "convrot_groupsize": 256}
|
||||
|
||||
|
||||
def _load(monkeypatch, spec, h3_markers=None):
|
||||
def _load(monkeypatch, spec, h3_markers=None, quantization=None, local_device="cpu"):
|
||||
"""Run load_comfyui_transformer with a dummy spec; return the load kwargs."""
|
||||
seen = {}
|
||||
|
||||
@@ -37,7 +37,7 @@ def _load(monkeypatch, spec, h3_markers=None):
|
||||
"resolve_minimax_h3_checkpoint_quantization",
|
||||
lambda markers: ("quant-config", dict(markers)),
|
||||
)
|
||||
monkeypatch.setattr(spec_mod, "get_local_torch_device", lambda: "cpu")
|
||||
monkeypatch.setattr(spec_mod, "get_local_torch_device", lambda: local_device)
|
||||
monkeypatch.setattr(
|
||||
"sglang.multimodal_gen.runtime.loader.transformer_load_utils."
|
||||
"resolve_transformer_gguf_to_load",
|
||||
@@ -45,7 +45,7 @@ def _load(monkeypatch, spec, h3_markers=None):
|
||||
)
|
||||
server_args = SimpleNamespace(
|
||||
model_paths={},
|
||||
quantization=None,
|
||||
quantization=quantization,
|
||||
nunchaku_config=None,
|
||||
hsdp_replicate_dim=1,
|
||||
hsdp_shard_dim=1,
|
||||
@@ -101,3 +101,43 @@ def test_h3_spec_still_rejects_non_int8_markers(monkeypatch):
|
||||
)
|
||||
with pytest.raises(ValueError, match="INT8 ConvRot"):
|
||||
_load(monkeypatch, h3, h3_markers={"x": {"format": "nvfp4"}})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("quantization", ["fp8", "convrot_int8"])
|
||||
def test_bf16_file_is_quantized_online(monkeypatch, quantization):
|
||||
"""A BF16 file with --quantization used to load with no quant config, so the
|
||||
flag was silently ignored and the DiT stayed in BF16."""
|
||||
hook_calls = []
|
||||
quant_spec = SimpleNamespace(
|
||||
quant_config="online-config",
|
||||
param_dtype=None,
|
||||
post_load_hooks=[hook_calls.append],
|
||||
is_convrot_int8=quantization == "convrot_int8",
|
||||
needs_device_weight_postprocess=True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"sglang.multimodal_gen.runtime.loader.transformer_load_utils."
|
||||
"resolve_transformer_quant_load_spec",
|
||||
lambda **kwargs: quant_spec,
|
||||
)
|
||||
seen = _load(
|
||||
monkeypatch,
|
||||
_spec(quant_markers=lambda paths: {}),
|
||||
quantization=quantization,
|
||||
local_device="gpu",
|
||||
)
|
||||
assert seen["init_params"]["quant_config"] == "online-config"
|
||||
assert seen["param_dtype"] is None
|
||||
assert len(hook_calls) == 1
|
||||
# Online FP8 quantizes on the GPU; only ConvRot INT8 can start from host weights.
|
||||
expected = "cpu" if quantization == "convrot_int8" else "gpu"
|
||||
assert str(seen["weight_load_plan"].checkpoint_load_device) == expected
|
||||
|
||||
|
||||
def test_bf16_file_rejects_methods_that_need_serialized_weights(monkeypatch):
|
||||
with pytest.raises(ValueError, match="pre-quantized"):
|
||||
_load(
|
||||
monkeypatch,
|
||||
_spec(quant_markers=lambda paths: {}),
|
||||
quantization="mxfp8",
|
||||
)
|
||||
|
||||
@@ -299,7 +299,7 @@ def _h3_runtime_that_must_not_start() -> SGLDiffusionGenerator:
|
||||
(
|
||||
"blocks.0.attn.to_q.weight",
|
||||
{"quantization": "convrot_int8"},
|
||||
"native model directories",
|
||||
"native model directory",
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
@@ -79,7 +79,9 @@ _GRAPH = {"execution_mode": "cuda_graph"}
|
||||
),
|
||||
("vdn_h3", {"attention_backend": "flash_attention"}, True, "does not apply"),
|
||||
("qwen_image", {"attention_backend": "sol_attention"}, False, "does not apply"),
|
||||
("flux", {"quantization": "fp8"}, False, "native model directories"),
|
||||
("flux", {"quantization": "fp8"}, False, "native model directory"),
|
||||
("qwen_image21", {"quantization": "mxfp8"}, False, "pre-quantized"),
|
||||
("qwen_image21", {"quantization": "fp8"}, False, "pre-quantized"),
|
||||
("lumina2", {"int8_gemm": "comfy_kitchen"}, False, "no effect"),
|
||||
("qwen_image", {"cache_mode": "cache_dit"}, False, "cache_dit is not"),
|
||||
("minimax_h3", {"cache_mode": "teacache"}, False, "teacache is not"),
|
||||
@@ -101,6 +103,8 @@ def test_preflight_rejects_what_the_model_cannot_run(
|
||||
("minimax_h3", {"int8_gemm": "comfy_kitchen", "cache_mode": "spectrum"}, False),
|
||||
("fast_h3", {"attention_backend": "video_sparse_attn_h3"}, True),
|
||||
("vdn_h3", {"quantization": "fp8", "cache_mode": "cache_dit"}, True),
|
||||
("qwen_image21", {"quantization": "convrot_int8"}, False),
|
||||
("qwen_image21", {"int8_gemm": "sglang_jit"}, False),
|
||||
],
|
||||
)
|
||||
def test_preflight_accepts_supported_combinations(
|
||||
|
||||
Reference in New Issue
Block a user