[Feat] ComfyUI: Add online INT8 quantization for BF16 Qwen-Image-2.1 single files (#44865)

Co-authored-by: WenhaoZhang <42087078+niehen6174@users.noreply.github.com>
This commit is contained in:
Avaya Aggarwal
2026-10-21 16:01:18 +08:00
committed by GitHub
co-authored by WenhaoZhang
parent a361f0c410
commit fee32a5c1f
7 changed files with 144 additions and 15 deletions
@@ -59,7 +59,7 @@ extension, which only changes the presentation.
| --- | --- |
| Parallel | `parallel_mode` auto or manual; manual sets `tp_size`, `ulysses_degree`, `ring_degree`, whose product must equal `num_gpus` |
| Attention | `attention_backend`: auto (model default), flash_attention, torch_sdpa, sage_attention2, sage_attention3, sol_attention (H3; needs the `sol-attn` package; picking it fills `attention_backend_config` with its defaults), and the H3 sparse backends; `attention_backend_config` |
| Quantization | `quantization` (online, native directories only); `int8_gemm`: GEMM implementation for INT8 ConvRot weights, auto / sglang_jit / comfy_kitchen |
| Quantization | `quantization` (online: native directories, or `convrot_int8` on a BF16 Qwen-Image-2.1 file); `int8_gemm`: GEMM implementation for INT8 ConvRot weights, auto / sglang_jit / comfy_kitchen |
| Cache | `cache_mode`: off, cache_dit, teacache, spectrum (lossy). For cache_dit, `cache_dit_tuning` auto keeps the defaults; manual sets warmup steps, residual threshold, max consecutive cached steps and Fn compute blocks |
| Execution | `execution_mode`: eager, torch_compile, cuda_graph (breakable CUDA graph); `regional_compile` |
| Memory | `memory_mode`: resident or DiT layerwise offload; `resident_layers`, `prefetch_layers` |
@@ -72,7 +72,7 @@ worker and raises instead of dropping a setting. Current limits in integrated mo
| cuda_graph | MiniMax H3, with `cache_mode` off and no layerwise offload |
| torch_compile | all but FastH3 and VDN-H3 |
| ring_degree > 1 | not FastH3 / VDN-H3; MiniMax H3 needs `fa` |
| quantization | native model directories; ComfyUI files carry their own format |
| quantization | native model directories; BF16 Qwen-Image-2.1 files with `convrot_int8`. Other ComfyUI files carry their own format |
| cache_dit, spectrum | MiniMax H3 family (spectrum: sequence-parallel degree 1); other models send each sampler step as its own request, so a step cache needs per-run state first |
| teacache | none yet |
@@ -133,6 +133,12 @@ _ATTENTION_BY_MODEL = {
}
# Model types whose BF16 ComfyUI file has been validated with online quantization.
_SINGLE_FILE_ONLINE_QUANT = frozenset({"qwen_image21"})
# Online FP8 drifts on Qwen-Image 2.1 (SSIM ~0.85 vs BF16), so only INT8 is allowed.
_SINGLE_FILE_ONLINE_METHODS = frozenset({"convrot_int8"})
def _unsupported(feature: str, model_type: str) -> ValueError:
return ValueError(
f"{feature} is not supported for {model_type} in ComfyUI integrated mode yet"
@@ -199,14 +205,25 @@ def _check_cache(model_type: str, opts: dict) -> None:
def _check_weights(model_type: str, opts: dict, *, native_dir: bool) -> None:
if opts.get("quantization") and not native_dir:
raise ValueError(
"quantization (online, at load) applies to native model directories; "
"ComfyUI files carry their own format, so load a pre-quantized file "
"(e.g. *_int8_convrot) instead"
)
quantization = opts.get("quantization")
if quantization and not native_dir:
if model_type not in _SINGLE_FILE_ONLINE_QUANT:
raise ValueError(
f"quantization (online, at load) of a ComfyUI file is supported for "
f"{sorted(_SINGLE_FILE_ONLINE_QUANT)}; for {model_type} use a native "
"model directory or a pre-quantized file (e.g. *_int8_convrot)"
)
if quantization not in _SINGLE_FILE_ONLINE_METHODS:
raise ValueError(
f"quantization={quantization} needs a pre-quantized checkpoint; a "
f"BF16 ComfyUI file takes one of {sorted(_SINGLE_FILE_ONLINE_METHODS)}"
)
kernel = opts.get("int8_kernel", "auto")
if kernel != "auto" and model_type not in H3_MODELS:
if (
kernel != "auto"
and model_type not in H3_MODELS
and model_type not in {"qwen_image21"}
):
raise ValueError(
f"int8_gemm={kernel} has no effect on {model_type}: only MiniMax H3 "
"runs ConvRot INT8 weights; leave it at auto"
@@ -55,7 +55,8 @@ _FIELDS = {
"quantization",
["none", "fp8", "convrot_int8", "mxfp8"],
"none",
"Online quantization; native model directories only.",
"Online quantization: native model directories, or a BF16 Qwen-Image 2.1 "
"file (convrot_int8).",
),
"int8_gemm": _field(
"quantization",
@@ -77,6 +77,9 @@ class ComfyUICheckpointSpec:
checkpoint_key_filter: Callable[[str], bool] | None = None
# Methods whose no-arg config quantizes BF16 weights after loading.
_ONLINE_QUANTIZATIONS = frozenset({"fp8", "mxfp4", "convrot_int8"})
_SPEC_REGISTRY: dict[str, ComfyUICheckpointSpec] = {}
_SPECS_DISCOVERED = False
@@ -191,6 +194,51 @@ def _load_comfyui_gguf_transformer(
model_cls.param_names_mapping = original_mapping
def _resolve_online_quant_spec(
*,
server_args: ServerArgs,
model_cls,
cls_name: str,
model_path: str,
dit_config,
):
from sglang.multimodal_gen.runtime.loader.transformer_load_utils import (
resolve_transformer_quant_load_spec,
)
if server_args.quantization not in _ONLINE_QUANTIZATIONS:
raise ValueError(
f"--quantization {server_args.quantization} needs a pre-quantized "
"checkpoint; a BF16 ComfyUI file can only be quantized online with "
f"one of {sorted(_ONLINE_QUANTIZATIONS)}"
)
return resolve_transformer_quant_load_spec(
hf_config={},
server_args=server_args,
safetensors_list=[model_path],
component_model_path=os.path.dirname(model_path) or ".",
model_cls=model_cls,
cls_name=cls_name,
component_name="transformer",
arch_config=dit_config.arch_config,
)
def _online_quant_load_plan(*, server_args: ServerArgs, quant_spec) -> WeightLoadPlan:
starts_on_cpu = server_args.should_start_component_on_cpu("transformer")
# Matches the native loader: only ConvRot INT8 quantizes from host weights.
checkpoint_load_device = (
torch.device("cpu")
if starts_on_cpu and quant_spec.is_convrot_int8
else get_local_torch_device()
)
return WeightLoadPlan.for_component(
checkpoint_load_device=checkpoint_load_device,
needs_device_weight_postprocess=quant_spec.needs_device_weight_postprocess,
component_starts_on_cpu=starts_on_cpu,
)
def load_comfyui_transformer(
pipeline: ComposedPipelineBase,
server_args: ServerArgs,
@@ -312,6 +360,23 @@ def load_comfyui_transformer(
adaln_curve_shape,
)
post_load_hooks = []
if quant_config is None and server_args.quantization is not None:
# A BF16 file has no markers; quantize it online like a native directory.
quant_spec = _resolve_online_quant_spec(
server_args=server_args,
model_cls=model_cls,
cls_name=spec.dit_cls_name,
model_path=model_path,
dit_config=dit_config,
)
quant_config = quant_spec.quant_config
param_dtype = quant_spec.param_dtype
post_load_hooks = quant_spec.post_load_hooks
weight_load_plan = _online_quant_load_plan(
server_args=server_args, quant_spec=quant_spec
)
logger.info(
"Loading %s from ComfyUI checkpoint %s, param_dtype: %s",
spec.dit_cls_name,
@@ -349,6 +414,8 @@ def load_comfyui_transformer(
)
finally:
model_cls.param_names_mapping = original_mapping
for post_load_hook in post_load_hooks:
post_load_hook(model)
for param in model.parameters():
param.requires_grad = False
@@ -10,7 +10,7 @@ from sglang.multimodal_gen.runtime.loader.comfyui_checkpoints import spec as spe
_MARKER = {"format": "int8_tensorwise", "convrot": True, "convrot_groupsize": 256}
def _load(monkeypatch, spec, h3_markers=None):
def _load(monkeypatch, spec, h3_markers=None, quantization=None, local_device="cpu"):
"""Run load_comfyui_transformer with a dummy spec; return the load kwargs."""
seen = {}
@@ -37,7 +37,7 @@ def _load(monkeypatch, spec, h3_markers=None):
"resolve_minimax_h3_checkpoint_quantization",
lambda markers: ("quant-config", dict(markers)),
)
monkeypatch.setattr(spec_mod, "get_local_torch_device", lambda: "cpu")
monkeypatch.setattr(spec_mod, "get_local_torch_device", lambda: local_device)
monkeypatch.setattr(
"sglang.multimodal_gen.runtime.loader.transformer_load_utils."
"resolve_transformer_gguf_to_load",
@@ -45,7 +45,7 @@ def _load(monkeypatch, spec, h3_markers=None):
)
server_args = SimpleNamespace(
model_paths={},
quantization=None,
quantization=quantization,
nunchaku_config=None,
hsdp_replicate_dim=1,
hsdp_shard_dim=1,
@@ -101,3 +101,43 @@ def test_h3_spec_still_rejects_non_int8_markers(monkeypatch):
)
with pytest.raises(ValueError, match="INT8 ConvRot"):
_load(monkeypatch, h3, h3_markers={"x": {"format": "nvfp4"}})
@pytest.mark.parametrize("quantization", ["fp8", "convrot_int8"])
def test_bf16_file_is_quantized_online(monkeypatch, quantization):
"""A BF16 file with --quantization used to load with no quant config, so the
flag was silently ignored and the DiT stayed in BF16."""
hook_calls = []
quant_spec = SimpleNamespace(
quant_config="online-config",
param_dtype=None,
post_load_hooks=[hook_calls.append],
is_convrot_int8=quantization == "convrot_int8",
needs_device_weight_postprocess=True,
)
monkeypatch.setattr(
"sglang.multimodal_gen.runtime.loader.transformer_load_utils."
"resolve_transformer_quant_load_spec",
lambda **kwargs: quant_spec,
)
seen = _load(
monkeypatch,
_spec(quant_markers=lambda paths: {}),
quantization=quantization,
local_device="gpu",
)
assert seen["init_params"]["quant_config"] == "online-config"
assert seen["param_dtype"] is None
assert len(hook_calls) == 1
# Online FP8 quantizes on the GPU; only ConvRot INT8 can start from host weights.
expected = "cpu" if quantization == "convrot_int8" else "gpu"
assert str(seen["weight_load_plan"].checkpoint_load_device) == expected
def test_bf16_file_rejects_methods_that_need_serialized_weights(monkeypatch):
with pytest.raises(ValueError, match="pre-quantized"):
_load(
monkeypatch,
_spec(quant_markers=lambda paths: {}),
quantization="mxfp8",
)
@@ -299,7 +299,7 @@ def _h3_runtime_that_must_not_start() -> SGLDiffusionGenerator:
(
"blocks.0.attn.to_q.weight",
{"quantization": "convrot_int8"},
"native model directories",
"native model directory",
),
],
)
@@ -79,7 +79,9 @@ _GRAPH = {"execution_mode": "cuda_graph"}
),
("vdn_h3", {"attention_backend": "flash_attention"}, True, "does not apply"),
("qwen_image", {"attention_backend": "sol_attention"}, False, "does not apply"),
("flux", {"quantization": "fp8"}, False, "native model directories"),
("flux", {"quantization": "fp8"}, False, "native model directory"),
("qwen_image21", {"quantization": "mxfp8"}, False, "pre-quantized"),
("qwen_image21", {"quantization": "fp8"}, False, "pre-quantized"),
("lumina2", {"int8_gemm": "comfy_kitchen"}, False, "no effect"),
("qwen_image", {"cache_mode": "cache_dit"}, False, "cache_dit is not"),
("minimax_h3", {"cache_mode": "teacache"}, False, "teacache is not"),
@@ -101,6 +103,8 @@ def test_preflight_rejects_what_the_model_cannot_run(
("minimax_h3", {"int8_gemm": "comfy_kitchen", "cache_mode": "spectrum"}, False),
("fast_h3", {"attention_backend": "video_sparse_attn_h3"}, True),
("vdn_h3", {"quantization": "fp8", "cache_mode": "cache_dit"}, True),
("qwen_image21", {"quantization": "convrot_int8"}, False),
("qwen_image21", {"int8_gemm": "sglang_jit"}, False),
],
)
def test_preflight_accepts_supported_combinations(