From 3831e7e0918052be50ef38b3ff62154573e213d7 Mon Sep 17 00:00:00 2001 From: Frederik Gossen Date: Sat, 10 Oct 2026 01:04:43 -0400 Subject: [PATCH] [diffusion] Z-Image: autotune the shared transformer block at both token scales (#35755) --- .../multimodal_gen/configs/models/dits/base.py | 4 ++++ .../multimodal_gen/configs/models/dits/zimage.py | 9 ++++++++- .../runtime/pipelines_core/stages/denoising.py | 5 +++++ .../multimodal_gen/runtime/utils/torch_compile.py | 14 ++++++++++++++ 4 files changed, 31 insertions(+), 1 deletion(-) diff --git a/python/sglang/multimodal_gen/configs/models/dits/base.py b/python/sglang/multimodal_gen/configs/models/dits/base.py index 3c2ce00abe..0c2ec84e20 100644 --- a/python/sglang/multimodal_gen/configs/models/dits/base.py +++ b/python/sglang/multimodal_gen/configs/models/dits/base.py @@ -46,6 +46,10 @@ class DiTConfig(ModelConfig): quant_config: QuantizationConfig | None = None torch_compile_mode: str = "max-autotune-no-cudagraphs" + # torch._inductor.config overrides for this model, applied before compiling, + # e.g. {"multi_kernel_hints": [64, 4096]}. + torch_compile_inductor_config: dict[str, Any] = field(default_factory=dict) + @staticmethod def add_cli_args(parser: Any, prefix: str = "dit-config") -> Any: """Add CLI arguments for DiTConfig fields""" diff --git a/python/sglang/multimodal_gen/configs/models/dits/zimage.py b/python/sglang/multimodal_gen/configs/models/dits/zimage.py index c58519231a..4f6262e2de 100644 --- a/python/sglang/multimodal_gen/configs/models/dits/zimage.py +++ b/python/sglang/multimodal_gen/configs/models/dits/zimage.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 from dataclasses import dataclass, field -from typing import Tuple +from typing import Any, Tuple from sglang.multimodal_gen.configs.models.dits.base import DiTArchConfig, DiTConfig @@ -96,3 +96,10 @@ class ZImageDitConfig(DiTConfig): arch_config: ZImageArchConfig = field(default_factory=ZImageArchConfig) prefix: str = "zimage" + + # One block serves both the caption tokens (tens) and the image and unified + # streams (thousands), with a dynamic token dim, so its GEMMs get autotuned + # for one branch and run for all three -- 1.4x slower when that goes wrong. + torch_compile_inductor_config: dict[str, Any] = field( + default_factory=lambda: {"multi_kernel_hints": [64, 4096]} + ) diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising.py index f3f20f2ead..57b1569554 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising.py @@ -173,6 +173,7 @@ from sglang.multimodal_gen.runtime.utils.precision import ( from sglang.multimodal_gen.runtime.utils.profiler import SGLDiffusionProfiler from sglang.multimodal_gen.runtime.utils.torch_compile import ( CompiledModuleRegistry, + apply_inductor_config, resolve_torch_compile_kwargs, ) @@ -584,6 +585,10 @@ class DenoisingStage(PipelineStage, RolloutDenoisingMixin): return dit_config = getattr(self.server_args.pipeline_config, "dit_config", None) + if not current_platform.is_npu(): + apply_inductor_config( + getattr(dit_config, "torch_compile_inductor_config", {}) + ) compile_kwargs, mode = resolve_torch_compile_kwargs( "SGLANG_TORCH_COMPILE_MODE", config=dit_config, diff --git a/python/sglang/multimodal_gen/runtime/utils/torch_compile.py b/python/sglang/multimodal_gen/runtime/utils/torch_compile.py index d67f350766..094fa2de8c 100644 --- a/python/sglang/multimodal_gen/runtime/utils/torch_compile.py +++ b/python/sglang/multimodal_gen/runtime/utils/torch_compile.py @@ -8,6 +8,9 @@ from typing import Any import torch.nn as nn from sglang.multimodal_gen.runtime.platforms import current_platform +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) def maybe_enable_inductor_compute_comm_overlap() -> None: @@ -19,6 +22,17 @@ def maybe_enable_inductor_compute_comm_overlap() -> None: pass +def apply_inductor_config(overrides: dict[str, Any]) -> None: + """Apply a model's torch._inductor.config overrides before it is compiled.""" + import torch._inductor.config as _inductor_cfg + + for name, value in overrides.items(): + if not hasattr(_inductor_cfg, name): + logger.warning("Unknown torch._inductor.config option %r, ignoring", name) + continue + setattr(_inductor_cfg, name, value) + + def build_torch_compile_kwargs( *, mode: str | None,