[diffusion] Z-Image: autotune the shared transformer block at both token scales (#35755)
This commit is contained in:
@@ -46,6 +46,10 @@ class DiTConfig(ModelConfig):
|
||||
quant_config: QuantizationConfig | None = None
|
||||
torch_compile_mode: str = "max-autotune-no-cudagraphs"
|
||||
|
||||
# torch._inductor.config overrides for this model, applied before compiling,
|
||||
# e.g. {"multi_kernel_hints": [64, 4096]}.
|
||||
torch_compile_inductor_config: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
@staticmethod
|
||||
def add_cli_args(parser: Any, prefix: str = "dit-config") -> Any:
|
||||
"""Add CLI arguments for DiTConfig fields"""
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Tuple
|
||||
from typing import Any, Tuple
|
||||
|
||||
from sglang.multimodal_gen.configs.models.dits.base import DiTArchConfig, DiTConfig
|
||||
|
||||
@@ -96,3 +96,10 @@ class ZImageDitConfig(DiTConfig):
|
||||
arch_config: ZImageArchConfig = field(default_factory=ZImageArchConfig)
|
||||
|
||||
prefix: str = "zimage"
|
||||
|
||||
# One block serves both the caption tokens (tens) and the image and unified
|
||||
# streams (thousands), with a dynamic token dim, so its GEMMs get autotuned
|
||||
# for one branch and run for all three -- 1.4x slower when that goes wrong.
|
||||
torch_compile_inductor_config: dict[str, Any] = field(
|
||||
default_factory=lambda: {"multi_kernel_hints": [64, 4096]}
|
||||
)
|
||||
|
||||
@@ -173,6 +173,7 @@ from sglang.multimodal_gen.runtime.utils.precision import (
|
||||
from sglang.multimodal_gen.runtime.utils.profiler import SGLDiffusionProfiler
|
||||
from sglang.multimodal_gen.runtime.utils.torch_compile import (
|
||||
CompiledModuleRegistry,
|
||||
apply_inductor_config,
|
||||
resolve_torch_compile_kwargs,
|
||||
)
|
||||
|
||||
@@ -584,6 +585,10 @@ class DenoisingStage(PipelineStage, RolloutDenoisingMixin):
|
||||
return
|
||||
|
||||
dit_config = getattr(self.server_args.pipeline_config, "dit_config", None)
|
||||
if not current_platform.is_npu():
|
||||
apply_inductor_config(
|
||||
getattr(dit_config, "torch_compile_inductor_config", {})
|
||||
)
|
||||
compile_kwargs, mode = resolve_torch_compile_kwargs(
|
||||
"SGLANG_TORCH_COMPILE_MODE",
|
||||
config=dit_config,
|
||||
|
||||
@@ -8,6 +8,9 @@ from typing import Any
|
||||
import torch.nn as nn
|
||||
|
||||
from sglang.multimodal_gen.runtime.platforms import current_platform
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def maybe_enable_inductor_compute_comm_overlap() -> None:
|
||||
@@ -19,6 +22,17 @@ def maybe_enable_inductor_compute_comm_overlap() -> None:
|
||||
pass
|
||||
|
||||
|
||||
def apply_inductor_config(overrides: dict[str, Any]) -> None:
|
||||
"""Apply a model's torch._inductor.config overrides before it is compiled."""
|
||||
import torch._inductor.config as _inductor_cfg
|
||||
|
||||
for name, value in overrides.items():
|
||||
if not hasattr(_inductor_cfg, name):
|
||||
logger.warning("Unknown torch._inductor.config option %r, ignoring", name)
|
||||
continue
|
||||
setattr(_inductor_cfg, name, value)
|
||||
|
||||
|
||||
def build_torch_compile_kwargs(
|
||||
*,
|
||||
mode: str | None,
|
||||
|
||||
Reference in New Issue
Block a user