[Perf] diffusion: Keep every component resident in auto mode when memory allows (#45978)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Mick
2026-10-24 09:26:13 +08:00
committed by GitHub
co-authored by Mick Qian Claude Opus 5.5
parent be8600d02d
commit 4e2d5b17ad
10 changed files with 441 additions and 75 deletions
@@ -288,9 +288,12 @@ class Wan2_2_T2V_A14B_Config(WanT2V480PConfig):
self.dit_config.torch_compile_mode = "default"
def get_model_deployment_config(self) -> ModelDeploymentConfig:
# both 14B experts plus T5 take ~66 GB per GPU; 120 keeps 80 GB cards streaming
return ModelDeploymentConfig(
dit_layerwise_offload_modes=("auto", "memory"),
auto_dit_offload_prefetch_size=2,
keep_resident_min_available_gb=120,
keep_resident_components=("dit", "vae"),
)
@@ -305,9 +308,12 @@ class Wan2_2_I2V_A14B_Config(WanI2V720PConfig):
self.dit_config.boundary_ratio = self.boundary_ratio
def get_model_deployment_config(self) -> ModelDeploymentConfig:
# both 14B experts plus T5 take ~66 GB per GPU; 120 keeps 80 GB cards streaming
return ModelDeploymentConfig(
dit_layerwise_offload_modes=("auto", "memory"),
auto_dit_offload_prefetch_size=2,
keep_resident_min_available_gb=120,
keep_resident_components=("dit", "vae"),
)
@@ -419,7 +419,9 @@ def checkpoint_bytes(model_path: str) -> int:
_DIT_SUBFOLDER = "transformer"
_DIT_SINGLE_FILE = "diffusion_pytorch_model.safetensors"
_FLOAT_DTYPES = frozenset({"F64", "F32", "F16", "BF16"})
_FLOAT_DTYPE_BYTES = {"F64": 8, "F32": 4, "F16": 2, "BF16": 2}
_FLOAT_DTYPES = frozenset(_FLOAT_DTYPE_BYTES)
_UNSIZED_WEIGHT_SUFFIXES = (".bin", ".pt", ".pth", ".ckpt", ".gguf")
def _index_shard_names(index_path: str) -> list[str]:
@@ -507,6 +509,59 @@ def dit_parameter_count(
return sum(counts.values())
def checkpoint_component_sizes(
model_path: str, *, subfolder: str | None, revision: str | None
) -> dict[str, tuple[int, int]] | None:
"""(parameter count, stored bytes) of each model_index.json component of a
locally cached Diffusers checkpoint, from safetensors headers.
Weightless components (tokenizers, schedulers) are left out. None when any
component cannot be sized from local float safetensors (not cached, missing
shards, other formats, quantized weights).
"""
try:
model_path = os.path.expanduser(model_path)
if os.path.isdir(model_path):
root = model_path
elif os.path.exists(model_path) or envs.SGLANG_USE_MODELSCOPE.get():
return None
else:
root = snapshot_download(
repo_id=model_path,
revision=revision,
allow_patterns=["*model_index.json"],
local_files_only=True,
)
if subfolder:
root = os.path.join(root, subfolder)
with open(os.path.join(root, "model_index.json")) as f:
components = [
name
for name, spec in json.load(f).items()
if isinstance(spec, list) and spec[0] is not None
]
sizes = {}
for name in components:
folder = os.path.join(root, name)
# a component folder not yet downloaded raises here
files = os.listdir(folder)
if not any(file.endswith(".safetensors") for file in files):
if any(file.endswith(_UNSIZED_WEIGHT_SUFFIXES) for file in files):
return None
continue
counts = _local_dit_parameter_counts(folder)
if not counts or not counts.keys() <= _FLOAT_DTYPES:
return None
sizes[name] = (
sum(counts.values()),
sum(n * _FLOAT_DTYPE_BYTES[dtype] for dtype, n in counts.items()),
)
return sizes
except Exception as exc:
logger.debug("Could not size the checkpoint of %s: %s", model_path, exc)
return None
def keep_checkpoint_mapped(*, weight_bytes: int, component: str) -> bool:
"""Whether a component's weights should stay on their file mapping.
@@ -18,6 +18,7 @@ from sglang.multimodal_gen.runtime.entrypoints.openai.realtime.registry import (
)
from sglang.multimodal_gen.runtime.loader.utils import (
BYTES_PER_GB,
checkpoint_component_sizes,
dit_parameter_count,
)
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload_components import (
@@ -26,12 +27,16 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload_co
LAYERWISE_OFFLOAD_IMAGE_ENCODER_GROUP,
LAYERWISE_OFFLOAD_TEXT_ENCODER_GROUP,
LAYERWISE_OFFLOAD_VAE_GROUP,
is_dit_component_name,
normalize_layerwise_offload_components,
)
from sglang.multimodal_gen.runtime.platforms import current_platform
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
from sglang.multimodal_gen.runtime.utils.model_overlay import resolve_model_overlay
from sglang.multimodal_gen.runtime.utils.precision import resolve_precision
from sglang.multimodal_gen.runtime.utils.precision import (
resolve_component_precision,
resolve_precision,
)
if TYPE_CHECKING:
from sglang.multimodal_gen.runtime.server_args.server_args import ServerArgs
@@ -40,6 +45,8 @@ logger = init_logger(__name__)
# Picked, not measured; what stays free holds activations and the other components.
AUTO_DIT_MAX_DEVICE_FRACTION = 0.8
# auto keeps every component resident once the least-free GPU holds all weights this many times
AUTO_ALL_RESIDENT_WEIGHT_MULTIPLE = 2
def _uses_svdquant(nunchaku_config) -> bool:
@@ -110,6 +117,12 @@ DEFAULT_LAYERWISE_COMPONENT_ARG_NAMES = (
# larger so it only stays resident on very-high-memory gpus
IMAGE_GEN_KEEP_RESIDENT_MIN_AVAILABLE_GB = 45.0
DEFAULT_KEEP_RESIDENT_MIN_AVAILABLE_GB = 120.0
ALL_OFFLOAD_COMPONENTS = (
LAYERWISE_OFFLOAD_DIT_GROUP,
LAYERWISE_OFFLOAD_TEXT_ENCODER_GROUP,
LAYERWISE_OFFLOAD_IMAGE_ENCODER_GROUP,
LAYERWISE_OFFLOAD_VAE_GROUP,
)
# torch's CPU allocator (mimalloc since 2.13) keeps freed pages in its arenas
@@ -141,6 +154,8 @@ class ServerArgsAutoTuner:
self.server_args = server_args
self._explicit_dit_residency = self._has_explicit_dit_residency()
self._dit_fit_checked = False
# memory-rich tier decision, resolved on first use
self._all_resident: bool | None = None
def _deployment_config(self) -> ModelDeploymentConfig:
return self.server_args.pipeline_config.get_model_deployment_config()
@@ -242,13 +257,18 @@ class ServerArgsAutoTuner:
disable_threshold_gb = self._resolve_keep_resident_min_available_gb(
deployment_config
)
if (
all_resident = self._keeps_all_components_resident()
if all_resident or (
min_available_gb is not None
and disable_threshold_gb is not None
and min_available_gb >= disable_threshold_gb
):
changed = []
components = set(deployment_config.keep_resident_components)
components = set(
ALL_OFFLOAD_COMPONENTS
if all_resident
else deployment_config.keep_resident_components
)
if args.pipeline_config.task_type.is_image_gen():
components.add(LAYERWISE_OFFLOAD_DIT_GROUP)
if (
@@ -713,6 +733,8 @@ class ServerArgsAutoTuner:
args = self.server_args
if args.performance_mode != "auto" or current_platform.is_cpu():
return components
if self._keeps_all_components_resident():
return []
deployment_config = self._deployment_config()
threshold_gb = self._resolve_keep_resident_min_available_gb(deployment_config)
@@ -749,6 +771,7 @@ class ServerArgsAutoTuner:
or envs.SGLANG_CACHE_DIT_ENABLED
or args.use_fsdp_inference
or args.explicit_residency_mode("transformer") is not None
or self._keeps_all_components_resident()
):
return False
@@ -757,6 +780,66 @@ class ServerArgsAutoTuner:
return True
return self._dit_overflows_device_in_auto(deployment_config)
def _keeps_all_components_resident(self) -> bool:
if self._all_resident is None:
self._all_resident = self._fits_all_components_resident()
return self._all_resident
def _fits_all_components_resident(self) -> bool:
"""Memory-rich auto tier: the least-free GPU holds every weight twice over.
Sized from the local checkpoint, so overrides, quantization and weights not
yet downloaded keep the per-model policy; one request's activations are
assumed, so batching and shared host/device pools are excluded too.
"""
args = self.server_args
if (
args.performance_mode != "auto"
or current_platform.is_cpu()
or current_platform.is_mps()
or current_platform.device_shares_host_memory()
or args.batching_max_size > 1
or args.quantization is not None
or args.transformer_weights_path is not None
or _uses_svdquant(args.nunchaku_config)
or args.component_paths
or args.component_weights_paths
or args.component_quantizations
or resolve_model_overlay(args.model_path) is not None
):
return False
available_gb = self._get_min_available_device_memory_gb()
if available_gb is None:
return False
sizes = checkpoint_component_sizes(
args.model_path, subfolder=args.model_subfolder, revision=args.revision
)
if sizes is None:
return False
weight_bytes = 0
for name, (num_params, stored_bytes) in sizes.items():
# loaded in the component's configured precision, else as stored
dtype = resolve_component_precision(args, name)
nbytes = num_params * dtype.itemsize if dtype is not None else stored_bytes
# tensor parallelism shards the DiT's linear layers across the ranks
weight_bytes += (
nbytes / (args.tp_size or 1) if is_dit_component_name(name) else nbytes
)
weight_gb = weight_bytes / BYTES_PER_GB
fits = available_gb >= AUTO_ALL_RESIDENT_WEIGHT_MULTIPLE * weight_gb
if fits:
logger.info(
"Auto memory policy for %s: %.1f GiB free on the least-free "
"selected GPU holds the %.1f GiB of weights per GPU at least "
"%dx, so every component without an explicit placement stays "
"resident.",
args.pipeline_config.__class__.__name__,
available_gb,
weight_gb,
AUTO_ALL_RESIDENT_WEIGHT_MULTIPLE,
)
return fits
def _dit_overflows_device_in_auto(
self, deployment_config: ModelDeploymentConfig
) -> bool:
@@ -326,7 +326,8 @@ ONE_GPU_CASES: list[DiffusionTestCase] = [
# request the stage regains that whole transfer, well past the
# non_denoise_stage tolerance. load_peak_vram carries the placed set, so
# it also pins that the placement happens at load, not on the first use.
# The encoder is already layerwise under the default component set.
# The encoder is pinned layerwise: on an 80 GB card auto would keep the
# whole 1.3B checkpoint resident.
# No consistency check: the lifetime moves weights, not math, so the
# output is the base case's and that case already guards it.
DiffusionTestCase(
@@ -334,6 +335,7 @@ ONE_GPU_CASES: list[DiffusionTestCase] = [
DiffusionServerArgs(
model_path=DEFAULT_WAN_2_1_T2V_1_3B_MODEL_NAME_FOR_TEST,
extras=[
"--component-residency text_encoder=layerwise-offload "
"--layerwise-resident-layers text_encoder=0.8 "
"--layerwise-residency-lifetime text_encoder=permanent",
],
@@ -272,10 +272,10 @@
"expected_avg_denoise_ms": 478.63,
"expected_median_denoise_ms": 443.42,
"estimated_full_test_time_s": 181.8,
"load_peak_vram_mb": 4920.72,
"runtime_peak_vram_mb": 9895.98,
"load_peak_allocated_mb": 4858.5,
"runtime_peak_allocated_mb": 6460.38
"load_peak_vram_mb": 25490.0,
"runtime_peak_vram_mb": 35568.0,
"load_peak_allocated_mb": 24622.71,
"runtime_peak_allocated_mb": 28881.55
},
"ideogram4_nvfp4_t2i": {
"expected_load_ms": 33439.28,
@@ -465,10 +465,10 @@
"expected_e2e_ms": 6693.01,
"expected_avg_denoise_ms": 128.85,
"expected_median_denoise_ms": 130.22,
"load_peak_vram_mb": 23594.0,
"runtime_peak_vram_mb": 28540.0,
"load_peak_allocated_mb": 23572.0,
"runtime_peak_allocated_mb": 25667.0,
"load_peak_vram_mb": 32296.0,
"runtime_peak_vram_mb": 36766.0,
"load_peak_allocated_mb": 32191.79,
"runtime_peak_allocated_mb": 34669.69,
"estimated_full_test_time_s": 80
},
"flux_2_image_t2i": {
@@ -850,10 +850,10 @@
"expected_e2e_ms": 4244.27,
"expected_avg_denoise_ms": 79.98,
"expected_median_denoise_ms": 81.01,
"load_peak_vram_mb": 23284.0,
"runtime_peak_vram_mb": 28148.0,
"load_peak_allocated_mb": 23262.0,
"runtime_peak_allocated_mb": 25541.0,
"load_peak_vram_mb": 27722.0,
"runtime_peak_vram_mb": 32332.0,
"load_peak_allocated_mb": 27651.29,
"runtime_peak_allocated_mb": 30128.94,
"estimated_full_test_time_s": 50.0
},
"zimage_image_t2i": {
@@ -880,10 +880,10 @@
"expected_e2e_ms": 906.14,
"expected_avg_denoise_ms": 82.61,
"expected_median_denoise_ms": 90.0,
"load_peak_vram_mb": 13120.0,
"runtime_peak_vram_mb": 17776.0,
"load_peak_allocated_mb": 12852.0,
"runtime_peak_allocated_mb": 15133.0,
"load_peak_vram_mb": 20546.0,
"runtime_peak_vram_mb": 24492.0,
"load_peak_allocated_mb": 20217.47,
"runtime_peak_allocated_mb": 22117.63,
"estimated_full_test_time_s": 50
},
"zimage_image_t2i_fp8": {
@@ -940,10 +940,10 @@
"expected_e2e_ms": 910.85,
"expected_avg_denoise_ms": 83.08,
"expected_median_denoise_ms": 89.93,
"load_peak_vram_mb": 14196.0,
"runtime_peak_vram_mb": 17612.0,
"load_peak_allocated_mb": 13059.0,
"runtime_peak_allocated_mb": 15307.0,
"load_peak_vram_mb": 21210.0,
"runtime_peak_vram_mb": 24626.0,
"load_peak_allocated_mb": 20453.97,
"runtime_peak_allocated_mb": 22291.56,
"estimated_full_test_time_s": 90
},
"zimage_image_t2i_2_gpus": {
@@ -970,10 +970,10 @@
"expected_e2e_ms": 752.84,
"expected_avg_denoise_ms": 51.13,
"expected_median_denoise_ms": 54.3,
"load_peak_vram_mb": 13120.0,
"runtime_peak_vram_mb": 17602.0,
"load_peak_allocated_mb": 12852.0,
"runtime_peak_allocated_mb": 15131.0,
"load_peak_vram_mb": 20546.0,
"runtime_peak_vram_mb": 24340.0,
"load_peak_allocated_mb": 20217.47,
"runtime_peak_allocated_mb": 22115.25,
"estimated_full_test_time_s": 50.0
},
"qwen_image_edit_ti2i": {
@@ -1246,10 +1246,10 @@
"expected_e2e_ms": 4739.06,
"expected_avg_denoise_ms": 76.99,
"expected_median_denoise_ms": 98.28,
"load_peak_vram_mb": 7814.0,
"runtime_peak_vram_mb": 18078.0,
"load_peak_allocated_mb": 3641.0,
"runtime_peak_allocated_mb": 9730.0,
"load_peak_vram_mb": 26126.0,
"runtime_peak_vram_mb": 33340.0,
"load_peak_allocated_mb": 25740.95,
"runtime_peak_allocated_mb": 31408.67,
"estimated_full_test_time_s": 60
},
"wan2_1_t2v_1.3b_encoder_permanent_residents": {
@@ -1389,10 +1389,10 @@
"expected_e2e_ms": 8106.87,
"expected_avg_denoise_ms": 143.82,
"expected_median_denoise_ms": 144.57,
"load_peak_vram_mb": 7814.0,
"runtime_peak_vram_mb": 18078.0,
"load_peak_allocated_mb": 3641.0,
"runtime_peak_allocated_mb": 9673.0,
"load_peak_vram_mb": 26126.0,
"runtime_peak_vram_mb": 33340.0,
"load_peak_allocated_mb": 25740.95,
"runtime_peak_allocated_mb": 31354.62,
"estimated_full_test_time_s": 150
},
"wan2_1_t2v_1.3b_cfg_parallel": {
@@ -1460,10 +1460,10 @@
"expected_e2e_ms": 4435.63,
"expected_avg_denoise_ms": 73.51,
"expected_median_denoise_ms": 74.05,
"load_peak_vram_mb": 5422.0,
"runtime_peak_vram_mb": 11972.0,
"load_peak_allocated_mb": 5274.0,
"runtime_peak_allocated_mb": 9251.0,
"load_peak_vram_mb": 15292.0,
"runtime_peak_vram_mb": 19728.0,
"load_peak_allocated_mb": 14906.86,
"runtime_peak_allocated_mb": 18094.7,
"estimated_full_test_time_s": 60.0
},
"turbo_wan2_1_t2v_1.3b": {
@@ -1629,10 +1629,10 @@
"expected_e2e_ms": 15599.56,
"expected_avg_denoise_ms": 253.29,
"expected_median_denoise_ms": 256.21,
"load_peak_vram_mb": 17610.0,
"runtime_peak_vram_mb": 39856.0,
"load_peak_allocated_mb": 11629.0,
"runtime_peak_allocated_mb": 26066.0,
"load_peak_vram_mb": 36644.0,
"runtime_peak_vram_mb": 53216.0,
"load_peak_allocated_mb": 36480.46,
"runtime_peak_allocated_mb": 49073.06,
"estimated_full_test_time_s": 80
},
"qwen_image_edit_2509_ti2i": {
@@ -1784,10 +1784,10 @@
"expected_e2e_ms": 3073.51,
"expected_avg_denoise_ms": 83.86,
"expected_median_denoise_ms": 112.67,
"load_peak_vram_mb": 17610.0,
"runtime_peak_vram_mb": 39054.0,
"load_peak_allocated_mb": 11629.0,
"runtime_peak_allocated_mb": 26051.0,
"load_peak_vram_mb": 36644.0,
"runtime_peak_vram_mb": 53636.0,
"load_peak_allocated_mb": 36480.46,
"runtime_peak_allocated_mb": 49057.2,
"estimated_full_test_time_s": 60
},
"fast_hunyuan_video": {
@@ -1812,11 +1812,11 @@
"expected_e2e_ms": 9123.68,
"expected_avg_denoise_ms": 871.39,
"expected_median_denoise_ms": 1010.9,
"load_peak_vram_mb": 27320.0,
"runtime_peak_vram_mb": 35926.0,
"load_peak_vram_mb": 40050.0,
"runtime_peak_vram_mb": 47480.0,
"warmup_peak_vram_mb": 56748.0,
"load_peak_allocated_mb": 26241.0,
"runtime_peak_allocated_mb": 31922.0,
"load_peak_allocated_mb": 39773.7,
"runtime_peak_allocated_mb": 44331.45,
"estimated_full_test_time_s": 80
},
"wan2_2_i2v_a14b_2gpu": {
@@ -2287,10 +2287,10 @@
"expected_e2e_ms": 8138.0,
"expected_avg_denoise_ms": 146.48,
"expected_median_denoise_ms": 147.68,
"load_peak_vram_mb": 7814.0,
"runtime_peak_vram_mb": 18350.0,
"load_peak_allocated_mb": 3641.0,
"runtime_peak_allocated_mb": 9999.0,
"load_peak_vram_mb": 26126.0,
"runtime_peak_vram_mb": 33442.0,
"load_peak_allocated_mb": 25740.95,
"runtime_peak_allocated_mb": 31666.73,
"estimated_full_test_time_s": 90
},
"wan2_1_t2v_1_3b_dynamic_lora_default_quality_1gpu": {
@@ -2557,10 +2557,10 @@
"expected_e2e_ms": 1042.43,
"expected_avg_denoise_ms": 82.26,
"expected_median_denoise_ms": 81.75,
"load_peak_vram_mb": 7176.0,
"runtime_peak_vram_mb": 12634.0,
"load_peak_allocated_mb": 7033.0,
"runtime_peak_allocated_mb": 9311.0,
"load_peak_vram_mb": 14558.0,
"runtime_peak_vram_mb": 19230.0,
"load_peak_allocated_mb": 14399.24,
"runtime_peak_allocated_mb": 16295.65,
"estimated_full_test_time_s": 40.0
},
"hunyuan3d_shape_gen": {
@@ -2700,10 +2700,10 @@
"expected_e2e_ms": 8273.83,
"expected_avg_denoise_ms": 146.61,
"expected_median_denoise_ms": 147.3,
"load_peak_vram_mb": 7814.0,
"runtime_peak_vram_mb": 18080.0,
"load_peak_allocated_mb": 3641.0,
"runtime_peak_allocated_mb": 9673.0,
"load_peak_vram_mb": 26126.0,
"runtime_peak_vram_mb": 34026.0,
"load_peak_allocated_mb": 25740.95,
"runtime_peak_allocated_mb": 31340.91,
"estimated_full_test_time_s": 70
},
"flux_2_image_t2i_upscaling_4x": {
@@ -2843,10 +2843,10 @@
"expected_e2e_ms": 8333.66,
"expected_avg_denoise_ms": 144.68,
"expected_median_denoise_ms": 144.74,
"load_peak_vram_mb": 7814.0,
"runtime_peak_vram_mb": 24380.0,
"load_peak_allocated_mb": 3641.0,
"runtime_peak_allocated_mb": 10508.0,
"load_peak_vram_mb": 26126.0,
"runtime_peak_vram_mb": 39714.0,
"load_peak_allocated_mb": 25740.95,
"runtime_peak_allocated_mb": 31340.91,
"estimated_full_test_time_s": 100
},
"wan2_1_t2v_1.3b_frame_interp_2x_upscaling_4x": {
@@ -2914,10 +2914,10 @@
"expected_e2e_ms": 8099.03,
"expected_avg_denoise_ms": 143.71,
"expected_median_denoise_ms": 144.67,
"load_peak_vram_mb": 7814.0,
"runtime_peak_vram_mb": 24388.0,
"load_peak_allocated_mb": 3641.0,
"runtime_peak_allocated_mb": 10531.0,
"load_peak_vram_mb": 26126.0,
"runtime_peak_vram_mb": 40406.0,
"load_peak_allocated_mb": 25740.95,
"runtime_peak_allocated_mb": 31340.91,
"estimated_full_test_time_s": 100
},
"ltx_2.3_one_stage_ti2v": {
@@ -127,6 +127,11 @@ def mock_cuda_platform(
"sglang.multimodal_gen.runtime.platforms.current_platform.enable_dit_layerwise_offload_by_default",
return_value=True,
),
# checkpoint sizing reads the local HF cache; tests that need it pass a size
patch(
"sglang.multimodal_gen.runtime.server_args.auto_tune.checkpoint_component_sizes",
return_value=None,
),
):
yield
@@ -1232,6 +1232,8 @@ class TestOffloadDefaults(_CudaPlatformTestCase):
memory_gb=80,
available_memory_gb=None,
dit_parameter_count=None,
checkpoint_sizes=None,
is_cuda=True,
kwargs=None,
):
def get_available_gpu_memory(device_id=0, **_kwargs):
@@ -1251,6 +1253,10 @@ class TestOffloadDefaults(_CudaPlatformTestCase):
else Mock(return_value=dit_parameter_count)
),
),
patch(
"sglang.multimodal_gen.runtime.server_args.auto_tune.checkpoint_component_sizes",
return_value=checkpoint_sizes,
),
patch(
"sglang.multimodal_gen.runtime.platforms.current_platform.is_cpu",
return_value=False,
@@ -1261,7 +1267,7 @@ class TestOffloadDefaults(_CudaPlatformTestCase):
),
patch(
"sglang.multimodal_gen.runtime.platforms.current_platform.is_cuda",
return_value=True,
return_value=is_cuda,
),
patch(
"sglang.multimodal_gen.runtime.platforms.current_platform.enable_dit_layerwise_offload_by_default",
@@ -2107,6 +2113,101 @@ class TestOffloadDefaults(_CudaPlatformTestCase):
["text_encoder", "image_encoder"],
)
def _zimage_turbo(
self, *, memory_gb=141, available_memory_gb=139, checkpoint_sizes=None, **kwargs
):
return self._from_dict_with_pipeline_config(
ZImagePipelineConfig(),
memory_gb=memory_gb,
available_memory_gb=available_memory_gb,
checkpoint_sizes=checkpoint_sizes,
kwargs={
"model_path": "Tongyi-MAI/Z-Image-Turbo",
"performance_mode": "auto",
**kwargs,
},
)
# (parameters, stored bytes) of Tongyi-MAI/Z-Image-Turbo: the DiT is stored in
# fp32 (22.9 GiB) and served in bf16, 19.3 GiB per GPU in all
ZIMAGE_TURBO_SIZES = {
"transformer": (6_154_908_736, 24_619_634_944),
"text_encoder": (4_022_468_096, 8_044_936_192),
"vae": (83_819_683, 335_278_732),
}
def _residency(self, args):
return {
name: args.residency_mode(name)
for name in ("transformer", "text_encoder", "image_encoder", "vae")
}
def test_auto_keeps_every_component_resident_when_weights_fit_twice(self):
args = self._zimage_turbo(checkpoint_sizes=self.ZIMAGE_TURBO_SIZES)
self.assertIsNone(args.layerwise_offload_components)
self.assertFalse(args.dit_cpu_offload)
self.assertFalse(args.text_encoder_cpu_offload)
self.assertEqual(set(self._residency(args).values()), {RESIDENT})
def test_auto_tier_sizes_weights_in_their_serving_precision(self):
# fits twice in 45 GiB at bf16; the fp32-stored bytes would not
args = self._zimage_turbo(
memory_gb=48,
available_memory_gb=45,
checkpoint_sizes=self.ZIMAGE_TURBO_SIZES,
)
self.assertEqual(set(self._residency(args).values()), {RESIDENT})
def test_auto_tier_leaves_consumer_gpus_on_their_policy(self):
for memory_gb, available_gb in ((32, 31), (24, 23)):
with self.subTest(memory_gb=memory_gb):
sized = self._zimage_turbo(
memory_gb=memory_gb,
available_memory_gb=available_gb,
checkpoint_sizes=self.ZIMAGE_TURBO_SIZES,
)
unsized = self._zimage_turbo(
memory_gb=memory_gb, available_memory_gb=available_gb
)
self.assertEqual(self._residency(sized), self._residency(unsized))
self.assertNotEqual(sized.residency_mode("text_encoder"), RESIDENT)
def test_auto_tier_skips_batching_shared_memory_and_unsized_checkpoints(self):
unsized = self._zimage_turbo()
self.assertEqual(unsized.residency_mode("text_encoder"), LAYERWISE_OFFLOAD)
batching = self._zimage_turbo(
checkpoint_sizes=self.ZIMAGE_TURBO_SIZES, batching_max_size=2
)
self.assertEqual(batching.residency_mode("text_encoder"), LAYERWISE_OFFLOAD)
with patch.object(
current_platform, "device_shares_host_memory", return_value=True
):
shared = self._zimage_turbo(checkpoint_sizes=self.ZIMAGE_TURBO_SIZES)
self.assertEqual(shared.residency_mode("text_encoder"), LAYERWISE_OFFLOAD)
def test_auto_tier_replaces_whole_component_offload_off_cuda(self):
# off CUDA auto has no layerwise defaults and offloads the encoder whole
unsized = self._from_dict_with_pipeline_config(
ZImagePipelineConfig(),
memory_gb=192,
available_memory_gb=190,
is_cuda=False,
kwargs={"model_path": "Tongyi-MAI/Z-Image-Turbo"},
)
sized = self._from_dict_with_pipeline_config(
ZImagePipelineConfig(),
memory_gb=192,
available_memory_gb=190,
checkpoint_sizes=self.ZIMAGE_TURBO_SIZES,
is_cuda=False,
kwargs={"model_path": "Tongyi-MAI/Z-Image-Turbo"},
)
self.assertTrue(unsized.text_encoder_cpu_offload)
self.assertFalse(sized.text_encoder_cpu_offload)
self.assertEqual(set(self._residency(sized).values()), {RESIDENT})
def test_auto_lingbot_keeps_dit_resident_on_h100(self):
args = self._from_dict_with_pipeline_config(
LingBotWorldCausalDMDConfig(),
@@ -2215,6 +2316,45 @@ class TestOffloadDefaults(_CudaPlatformTestCase):
["dit", "text_encoder", "image_encoder", "vae"],
)
def test_auto_wan2_2_a14b_keeps_both_experts_resident_above_120gb(self):
for pipeline_config, model_path in (
(Wan2_2_T2V_A14B_Config(), "Wan-AI/Wan2.2-T2V-A14B-Diffusers"),
(Wan2_2_I2V_A14B_Config(), "Wan-AI/Wan2.2-I2V-A14B-Diffusers"),
):
with self.subTest(pipeline_config=pipeline_config.__class__.__name__):
args = self._from_dict_with_pipeline_config(
pipeline_config,
memory_gb=141,
available_memory_gb=139,
kwargs={"model_path": model_path, "performance_mode": "auto"},
)
self.assertFalse(args.dit_cpu_offload)
self.assertEqual(args.residency_mode("transformer"), RESIDENT)
self.assertEqual(args.residency_mode("transformer_2"), RESIDENT)
self.assertEqual(args.residency_mode("vae"), RESIDENT)
self.assertEqual(
args.layerwise_offload_components,
["text_encoder", "image_encoder"],
)
# two 14B experts plus T5 fit twice over: the encoder joins them
rich = self._from_dict_with_pipeline_config(
pipeline_config,
memory_gb=288,
available_memory_gb=280,
checkpoint_sizes={
"transformer": (14_288_901_184, 57_155_604_736),
"transformer_2": (14_288_901_184, 57_155_604_736),
"text_encoder": (5_680_910_336, 22_723_641_344),
"vae": (126_892_531, 507_570_124),
},
kwargs={"model_path": model_path, "performance_mode": "auto"},
)
self.assertIsNone(rich.layerwise_offload_components)
self.assertEqual(rich.residency_mode("text_encoder"), RESIDENT)
self.assertEqual(rich.residency_mode("transformer_2"), RESIDENT)
def test_auto_wan2_1_14b_keeps_dit_resident_on_h100(self):
for pipeline_config, model_path in (
(WanT2V720PConfig(), "Wan-AI/Wan2.1-T2V-14B-Diffusers"),
@@ -2,6 +2,7 @@
"""Unit tests for multimodal weight-loading utilities."""
import os
import shutil
import tempfile
import unittest
from pathlib import Path
@@ -15,6 +16,7 @@ from safetensors.torch import save_file as safetensors_save_file
from sglang.multimodal_gen.runtime.loader.utils import (
_list_safetensors_files,
checkpoint_bytes,
checkpoint_component_sizes,
dit_parameter_count,
)
from sglang.multimodal_gen.runtime.loader.weight_utils import (
@@ -266,6 +268,79 @@ class TestDitParameterCount(CustomTestCase):
self.assertEqual(hub_download.called, use_modelscope == "0")
def _write_pipeline(root: Path) -> None:
"""A Diffusers layout: a 58-byte DiT, a 16-byte encoder and weightless parts."""
_write_sharded_transformer(root)
(root / "text_encoder").mkdir()
safetensors_save_file(
{"e": torch.zeros(4)}, root / "text_encoder" / "model.safetensors"
)
for weightless in ("tokenizer", "scheduler"):
(root / weightless).mkdir()
(root / weightless / "config.json").write_text("{}")
(root / "model_index.json").write_text(
'{"_class_name": "P", "transformer": ["diffusers", "T"], '
'"text_encoder": ["transformers", "E"], "tokenizer": ["transformers", "K"], '
'"scheduler": ["diffusers", "S"], "safety_checker": [null, null], '
'"boundary_ratio": 0.9}'
)
class TestCheckpointComponentSizes(CustomTestCase):
SIZES = {"transformer": (17, 58), "text_encoder": (4, 16)}
def test_sizes_every_weighted_component(self):
with tempfile.TemporaryDirectory() as tmpdir:
_write_pipeline(Path(tmpdir) / "sub")
self.assertEqual(
checkpoint_component_sizes(tmpdir, subfolder="sub", revision=None),
self.SIZES,
)
def test_cached_hub_snapshot_is_sized_without_the_network(self):
with tempfile.TemporaryDirectory() as tmpdir:
_write_pipeline(Path(tmpdir))
with patch(_SNAPSHOT_DOWNLOAD, return_value=tmpdir) as snapshot_download:
self.assertEqual(
checkpoint_component_sizes(
"org/model", subfolder=None, revision="abc"
),
self.SIZES,
)
self.assertTrue(snapshot_download.call_args.kwargs["local_files_only"])
def test_partial_download_other_formats_or_quantized_weights_are_unsized(self):
cases = {
# only model_index.json fetched so far, as before the worker downloads
"index only": lambda root: shutil.rmtree(root / "text_encoder"),
"missing shard": lambda root: (
root / "transformer" / "shard-2.safetensors"
).unlink(),
"bin weights": lambda root: (
(root / "text_encoder" / "model.safetensors").unlink(),
(root / "text_encoder" / "pytorch_model.bin").write_bytes(b"0"),
),
"quantized": lambda root: safetensors_save_file(
{"q": torch.zeros(4, dtype=torch.float8_e4m3fn)},
root / "text_encoder" / "model.safetensors",
),
}
for name, damage in cases.items():
with self.subTest(name), tempfile.TemporaryDirectory() as tmpdir:
_write_pipeline(Path(tmpdir))
damage(Path(tmpdir))
self.assertIsNone(
checkpoint_component_sizes(tmpdir, subfolder=None, revision=None)
)
def test_uncached_repo_is_unsized(self):
with patch(_SNAPSHOT_DOWNLOAD, side_effect=LocalEntryNotFoundError("miss")):
self.assertIsNone(
checkpoint_component_sizes("org/model", subfolder=None, revision=None)
)
class TestDisableRunaiStreamerRankDiscoveryCollective(unittest.TestCase):
def test_never_touches_torch_distributed_even_when_initialized(self):
from runai_model_streamer.distributed_streamer.distributed_streamer import (