From fee32a5c1fbb47a7c78a2b75decff64037d2ef9b Mon Sep 17 00:00:00 2001 From: Avaya Aggarwal <119044997+OnePunchMonk@users.noreply.github.com> Date: Wed, 21 Oct 2026 13:31:18 +0530 Subject: [PATCH] [Feat] ComfyUI: Add online INT8 quantization for BF16 Qwen-Image-2.1 single files (#44865) Co-authored-by: WenhaoZhang <42087078+niehen6174@users.noreply.github.com> --- .../apps/ComfyUI_SGLDiffusion/README.md | 4 +- .../ComfyUI_SGLDiffusion/core/preflight.py | 31 +++++++-- .../apps/ComfyUI_SGLDiffusion/options.py | 3 +- .../loader/comfyui_checkpoints/spec.py | 67 +++++++++++++++++++ .../test/unit/test_comfyui_checkpoint_spec.py | 46 ++++++++++++- .../test/unit/test_comfyui_generator.py | 2 +- .../test/unit/test_comfyui_options.py | 6 +- 7 files changed, 144 insertions(+), 15 deletions(-) diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/README.md b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/README.md index 4959799c5e..cc047eba8b 100644 --- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/README.md +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/README.md @@ -59,7 +59,7 @@ extension, which only changes the presentation. | --- | --- | | Parallel | `parallel_mode` auto or manual; manual sets `tp_size`, `ulysses_degree`, `ring_degree`, whose product must equal `num_gpus` | | Attention | `attention_backend`: auto (model default), flash_attention, torch_sdpa, sage_attention2, sage_attention3, sol_attention (H3; needs the `sol-attn` package; picking it fills `attention_backend_config` with its defaults), and the H3 sparse backends; `attention_backend_config` | -| Quantization | `quantization` (online, native directories only); `int8_gemm`: GEMM implementation for INT8 ConvRot weights, auto / sglang_jit / comfy_kitchen | +| Quantization | `quantization` (online: native directories, or `convrot_int8` on a BF16 Qwen-Image-2.1 file); `int8_gemm`: GEMM implementation for INT8 ConvRot weights, auto / sglang_jit / comfy_kitchen | | Cache | `cache_mode`: off, cache_dit, teacache, spectrum (lossy). For cache_dit, `cache_dit_tuning` auto keeps the defaults; manual sets warmup steps, residual threshold, max consecutive cached steps and Fn compute blocks | | Execution | `execution_mode`: eager, torch_compile, cuda_graph (breakable CUDA graph); `regional_compile` | | Memory | `memory_mode`: resident or DiT layerwise offload; `resident_layers`, `prefetch_layers` | @@ -72,7 +72,7 @@ worker and raises instead of dropping a setting. Current limits in integrated mo | cuda_graph | MiniMax H3, with `cache_mode` off and no layerwise offload | | torch_compile | all but FastH3 and VDN-H3 | | ring_degree > 1 | not FastH3 / VDN-H3; MiniMax H3 needs `fa` | -| quantization | native model directories; ComfyUI files carry their own format | +| quantization | native model directories; BF16 Qwen-Image-2.1 files with `convrot_int8`. Other ComfyUI files carry their own format | | cache_dit, spectrum | MiniMax H3 family (spectrum: sequence-parallel degree 1); other models send each sampler step as its own request, so a step cache needs per-run state first | | teacache | none yet | diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/preflight.py b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/preflight.py index 7f66104f0e..2fc9366ee0 100644 --- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/preflight.py +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/preflight.py @@ -133,6 +133,12 @@ _ATTENTION_BY_MODEL = { } +# Model types whose BF16 ComfyUI file has been validated with online quantization. +_SINGLE_FILE_ONLINE_QUANT = frozenset({"qwen_image21"}) +# Online FP8 drifts on Qwen-Image 2.1 (SSIM ~0.85 vs BF16), so only INT8 is allowed. +_SINGLE_FILE_ONLINE_METHODS = frozenset({"convrot_int8"}) + + def _unsupported(feature: str, model_type: str) -> ValueError: return ValueError( f"{feature} is not supported for {model_type} in ComfyUI integrated mode yet" @@ -199,14 +205,25 @@ def _check_cache(model_type: str, opts: dict) -> None: def _check_weights(model_type: str, opts: dict, *, native_dir: bool) -> None: - if opts.get("quantization") and not native_dir: - raise ValueError( - "quantization (online, at load) applies to native model directories; " - "ComfyUI files carry their own format, so load a pre-quantized file " - "(e.g. *_int8_convrot) instead" - ) + quantization = opts.get("quantization") + if quantization and not native_dir: + if model_type not in _SINGLE_FILE_ONLINE_QUANT: + raise ValueError( + f"quantization (online, at load) of a ComfyUI file is supported for " + f"{sorted(_SINGLE_FILE_ONLINE_QUANT)}; for {model_type} use a native " + "model directory or a pre-quantized file (e.g. *_int8_convrot)" + ) + if quantization not in _SINGLE_FILE_ONLINE_METHODS: + raise ValueError( + f"quantization={quantization} needs a pre-quantized checkpoint; a " + f"BF16 ComfyUI file takes one of {sorted(_SINGLE_FILE_ONLINE_METHODS)}" + ) kernel = opts.get("int8_kernel", "auto") - if kernel != "auto" and model_type not in H3_MODELS: + if ( + kernel != "auto" + and model_type not in H3_MODELS + and model_type not in {"qwen_image21"} + ): raise ValueError( f"int8_gemm={kernel} has no effect on {model_type}: only MiniMax H3 " "runs ConvRot INT8 weights; leave it at auto" diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/options.py b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/options.py index f5f3075226..a2a482921e 100644 --- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/options.py +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/options.py @@ -55,7 +55,8 @@ _FIELDS = { "quantization", ["none", "fp8", "convrot_int8", "mxfp8"], "none", - "Online quantization; native model directories only.", + "Online quantization: native model directories, or a BF16 Qwen-Image 2.1 " + "file (convrot_int8).", ), "int8_gemm": _field( "quantization", diff --git a/python/sglang/multimodal_gen/runtime/loader/comfyui_checkpoints/spec.py b/python/sglang/multimodal_gen/runtime/loader/comfyui_checkpoints/spec.py index acfa5441f0..ad4a7e6b8f 100644 --- a/python/sglang/multimodal_gen/runtime/loader/comfyui_checkpoints/spec.py +++ b/python/sglang/multimodal_gen/runtime/loader/comfyui_checkpoints/spec.py @@ -77,6 +77,9 @@ class ComfyUICheckpointSpec: checkpoint_key_filter: Callable[[str], bool] | None = None +# Methods whose no-arg config quantizes BF16 weights after loading. +_ONLINE_QUANTIZATIONS = frozenset({"fp8", "mxfp4", "convrot_int8"}) + _SPEC_REGISTRY: dict[str, ComfyUICheckpointSpec] = {} _SPECS_DISCOVERED = False @@ -191,6 +194,51 @@ def _load_comfyui_gguf_transformer( model_cls.param_names_mapping = original_mapping +def _resolve_online_quant_spec( + *, + server_args: ServerArgs, + model_cls, + cls_name: str, + model_path: str, + dit_config, +): + from sglang.multimodal_gen.runtime.loader.transformer_load_utils import ( + resolve_transformer_quant_load_spec, + ) + + if server_args.quantization not in _ONLINE_QUANTIZATIONS: + raise ValueError( + f"--quantization {server_args.quantization} needs a pre-quantized " + "checkpoint; a BF16 ComfyUI file can only be quantized online with " + f"one of {sorted(_ONLINE_QUANTIZATIONS)}" + ) + return resolve_transformer_quant_load_spec( + hf_config={}, + server_args=server_args, + safetensors_list=[model_path], + component_model_path=os.path.dirname(model_path) or ".", + model_cls=model_cls, + cls_name=cls_name, + component_name="transformer", + arch_config=dit_config.arch_config, + ) + + +def _online_quant_load_plan(*, server_args: ServerArgs, quant_spec) -> WeightLoadPlan: + starts_on_cpu = server_args.should_start_component_on_cpu("transformer") + # Matches the native loader: only ConvRot INT8 quantizes from host weights. + checkpoint_load_device = ( + torch.device("cpu") + if starts_on_cpu and quant_spec.is_convrot_int8 + else get_local_torch_device() + ) + return WeightLoadPlan.for_component( + checkpoint_load_device=checkpoint_load_device, + needs_device_weight_postprocess=quant_spec.needs_device_weight_postprocess, + component_starts_on_cpu=starts_on_cpu, + ) + + def load_comfyui_transformer( pipeline: ComposedPipelineBase, server_args: ServerArgs, @@ -312,6 +360,23 @@ def load_comfyui_transformer( adaln_curve_shape, ) + post_load_hooks = [] + if quant_config is None and server_args.quantization is not None: + # A BF16 file has no markers; quantize it online like a native directory. + quant_spec = _resolve_online_quant_spec( + server_args=server_args, + model_cls=model_cls, + cls_name=spec.dit_cls_name, + model_path=model_path, + dit_config=dit_config, + ) + quant_config = quant_spec.quant_config + param_dtype = quant_spec.param_dtype + post_load_hooks = quant_spec.post_load_hooks + weight_load_plan = _online_quant_load_plan( + server_args=server_args, quant_spec=quant_spec + ) + logger.info( "Loading %s from ComfyUI checkpoint %s, param_dtype: %s", spec.dit_cls_name, @@ -349,6 +414,8 @@ def load_comfyui_transformer( ) finally: model_cls.param_names_mapping = original_mapping + for post_load_hook in post_load_hooks: + post_load_hook(model) for param in model.parameters(): param.requires_grad = False diff --git a/python/sglang/multimodal_gen/test/unit/test_comfyui_checkpoint_spec.py b/python/sglang/multimodal_gen/test/unit/test_comfyui_checkpoint_spec.py index 8ae973a560..1c82b233fc 100644 --- a/python/sglang/multimodal_gen/test/unit/test_comfyui_checkpoint_spec.py +++ b/python/sglang/multimodal_gen/test/unit/test_comfyui_checkpoint_spec.py @@ -10,7 +10,7 @@ from sglang.multimodal_gen.runtime.loader.comfyui_checkpoints import spec as spe _MARKER = {"format": "int8_tensorwise", "convrot": True, "convrot_groupsize": 256} -def _load(monkeypatch, spec, h3_markers=None): +def _load(monkeypatch, spec, h3_markers=None, quantization=None, local_device="cpu"): """Run load_comfyui_transformer with a dummy spec; return the load kwargs.""" seen = {} @@ -37,7 +37,7 @@ def _load(monkeypatch, spec, h3_markers=None): "resolve_minimax_h3_checkpoint_quantization", lambda markers: ("quant-config", dict(markers)), ) - monkeypatch.setattr(spec_mod, "get_local_torch_device", lambda: "cpu") + monkeypatch.setattr(spec_mod, "get_local_torch_device", lambda: local_device) monkeypatch.setattr( "sglang.multimodal_gen.runtime.loader.transformer_load_utils." "resolve_transformer_gguf_to_load", @@ -45,7 +45,7 @@ def _load(monkeypatch, spec, h3_markers=None): ) server_args = SimpleNamespace( model_paths={}, - quantization=None, + quantization=quantization, nunchaku_config=None, hsdp_replicate_dim=1, hsdp_shard_dim=1, @@ -101,3 +101,43 @@ def test_h3_spec_still_rejects_non_int8_markers(monkeypatch): ) with pytest.raises(ValueError, match="INT8 ConvRot"): _load(monkeypatch, h3, h3_markers={"x": {"format": "nvfp4"}}) + + +@pytest.mark.parametrize("quantization", ["fp8", "convrot_int8"]) +def test_bf16_file_is_quantized_online(monkeypatch, quantization): + """A BF16 file with --quantization used to load with no quant config, so the + flag was silently ignored and the DiT stayed in BF16.""" + hook_calls = [] + quant_spec = SimpleNamespace( + quant_config="online-config", + param_dtype=None, + post_load_hooks=[hook_calls.append], + is_convrot_int8=quantization == "convrot_int8", + needs_device_weight_postprocess=True, + ) + monkeypatch.setattr( + "sglang.multimodal_gen.runtime.loader.transformer_load_utils." + "resolve_transformer_quant_load_spec", + lambda **kwargs: quant_spec, + ) + seen = _load( + monkeypatch, + _spec(quant_markers=lambda paths: {}), + quantization=quantization, + local_device="gpu", + ) + assert seen["init_params"]["quant_config"] == "online-config" + assert seen["param_dtype"] is None + assert len(hook_calls) == 1 + # Online FP8 quantizes on the GPU; only ConvRot INT8 can start from host weights. + expected = "cpu" if quantization == "convrot_int8" else "gpu" + assert str(seen["weight_load_plan"].checkpoint_load_device) == expected + + +def test_bf16_file_rejects_methods_that_need_serialized_weights(monkeypatch): + with pytest.raises(ValueError, match="pre-quantized"): + _load( + monkeypatch, + _spec(quant_markers=lambda paths: {}), + quantization="mxfp8", + ) diff --git a/python/sglang/multimodal_gen/test/unit/test_comfyui_generator.py b/python/sglang/multimodal_gen/test/unit/test_comfyui_generator.py index aedc82201e..9f9d53020d 100644 --- a/python/sglang/multimodal_gen/test/unit/test_comfyui_generator.py +++ b/python/sglang/multimodal_gen/test/unit/test_comfyui_generator.py @@ -299,7 +299,7 @@ def _h3_runtime_that_must_not_start() -> SGLDiffusionGenerator: ( "blocks.0.attn.to_q.weight", {"quantization": "convrot_int8"}, - "native model directories", + "native model directory", ), ], ) diff --git a/python/sglang/multimodal_gen/test/unit/test_comfyui_options.py b/python/sglang/multimodal_gen/test/unit/test_comfyui_options.py index 508488458d..a961ed09c5 100644 --- a/python/sglang/multimodal_gen/test/unit/test_comfyui_options.py +++ b/python/sglang/multimodal_gen/test/unit/test_comfyui_options.py @@ -79,7 +79,9 @@ _GRAPH = {"execution_mode": "cuda_graph"} ), ("vdn_h3", {"attention_backend": "flash_attention"}, True, "does not apply"), ("qwen_image", {"attention_backend": "sol_attention"}, False, "does not apply"), - ("flux", {"quantization": "fp8"}, False, "native model directories"), + ("flux", {"quantization": "fp8"}, False, "native model directory"), + ("qwen_image21", {"quantization": "mxfp8"}, False, "pre-quantized"), + ("qwen_image21", {"quantization": "fp8"}, False, "pre-quantized"), ("lumina2", {"int8_gemm": "comfy_kitchen"}, False, "no effect"), ("qwen_image", {"cache_mode": "cache_dit"}, False, "cache_dit is not"), ("minimax_h3", {"cache_mode": "teacache"}, False, "teacache is not"), @@ -101,6 +103,8 @@ def test_preflight_rejects_what_the_model_cannot_run( ("minimax_h3", {"int8_gemm": "comfy_kitchen", "cache_mode": "spectrum"}, False), ("fast_h3", {"attention_backend": "video_sparse_attn_h3"}, True), ("vdn_h3", {"quantization": "fp8", "cache_mode": "cache_dit"}, True), + ("qwen_image21", {"quantization": "convrot_int8"}, False), + ("qwen_image21", {"int8_gemm": "sglang_jit"}, False), ], ) def test_preflight_accepts_supported_combinations(