diff --git a/docs/docs/sglang-diffusion/deployment_cookbook.mdx b/docs/docs/sglang-diffusion/deployment_cookbook.mdx
index d7f911c29b..2a7c3334e0 100644
--- a/docs/docs/sglang-diffusion/deployment_cookbook.mdx
+++ b/docs/docs/sglang-diffusion/deployment_cookbook.mdx
@@ -115,7 +115,7 @@ See [OpenAI API: Served model name](/docs/sglang-diffusion/api/openai_api#served
| `auto` |
- Default. Keeps legacy safe offload defaults and uses FSDP/CFG only on validated multi-GPU deployments where FSDP can replace DiT offload. |
+ Default. Keeps every component resident when the least-free GPU has at least twice the model's per-GPU weights free (sized from the cached checkpoint; not with batching or on shared-memory devices). Otherwise keeps legacy safe offload defaults and uses FSDP/CFG only on validated multi-GPU deployments where FSDP can replace DiT offload. |
| `speed` |
diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/wan.py b/python/sglang/multimodal_gen/configs/pipeline_configs/wan.py
index 711a1bd180..5421cdc5c9 100644
--- a/python/sglang/multimodal_gen/configs/pipeline_configs/wan.py
+++ b/python/sglang/multimodal_gen/configs/pipeline_configs/wan.py
@@ -288,9 +288,12 @@ class Wan2_2_T2V_A14B_Config(WanT2V480PConfig):
self.dit_config.torch_compile_mode = "default"
def get_model_deployment_config(self) -> ModelDeploymentConfig:
+ # both 14B experts plus T5 take ~66 GB per GPU; 120 keeps 80 GB cards streaming
return ModelDeploymentConfig(
dit_layerwise_offload_modes=("auto", "memory"),
auto_dit_offload_prefetch_size=2,
+ keep_resident_min_available_gb=120,
+ keep_resident_components=("dit", "vae"),
)
@@ -305,9 +308,12 @@ class Wan2_2_I2V_A14B_Config(WanI2V720PConfig):
self.dit_config.boundary_ratio = self.boundary_ratio
def get_model_deployment_config(self) -> ModelDeploymentConfig:
+ # both 14B experts plus T5 take ~66 GB per GPU; 120 keeps 80 GB cards streaming
return ModelDeploymentConfig(
dit_layerwise_offload_modes=("auto", "memory"),
auto_dit_offload_prefetch_size=2,
+ keep_resident_min_available_gb=120,
+ keep_resident_components=("dit", "vae"),
)
diff --git a/python/sglang/multimodal_gen/runtime/loader/utils.py b/python/sglang/multimodal_gen/runtime/loader/utils.py
index f38f006126..0389a191ab 100644
--- a/python/sglang/multimodal_gen/runtime/loader/utils.py
+++ b/python/sglang/multimodal_gen/runtime/loader/utils.py
@@ -419,7 +419,9 @@ def checkpoint_bytes(model_path: str) -> int:
_DIT_SUBFOLDER = "transformer"
_DIT_SINGLE_FILE = "diffusion_pytorch_model.safetensors"
-_FLOAT_DTYPES = frozenset({"F64", "F32", "F16", "BF16"})
+_FLOAT_DTYPE_BYTES = {"F64": 8, "F32": 4, "F16": 2, "BF16": 2}
+_FLOAT_DTYPES = frozenset(_FLOAT_DTYPE_BYTES)
+_UNSIZED_WEIGHT_SUFFIXES = (".bin", ".pt", ".pth", ".ckpt", ".gguf")
def _index_shard_names(index_path: str) -> list[str]:
@@ -507,6 +509,59 @@ def dit_parameter_count(
return sum(counts.values())
+def checkpoint_component_sizes(
+ model_path: str, *, subfolder: str | None, revision: str | None
+) -> dict[str, tuple[int, int]] | None:
+ """(parameter count, stored bytes) of each model_index.json component of a
+ locally cached Diffusers checkpoint, from safetensors headers.
+
+ Weightless components (tokenizers, schedulers) are left out. None when any
+ component cannot be sized from local float safetensors (not cached, missing
+ shards, other formats, quantized weights).
+ """
+ try:
+ model_path = os.path.expanduser(model_path)
+ if os.path.isdir(model_path):
+ root = model_path
+ elif os.path.exists(model_path) or envs.SGLANG_USE_MODELSCOPE.get():
+ return None
+ else:
+ root = snapshot_download(
+ repo_id=model_path,
+ revision=revision,
+ allow_patterns=["*model_index.json"],
+ local_files_only=True,
+ )
+ if subfolder:
+ root = os.path.join(root, subfolder)
+ with open(os.path.join(root, "model_index.json")) as f:
+ components = [
+ name
+ for name, spec in json.load(f).items()
+ if isinstance(spec, list) and spec[0] is not None
+ ]
+ sizes = {}
+ for name in components:
+ folder = os.path.join(root, name)
+ # a component folder not yet downloaded raises here
+ files = os.listdir(folder)
+ if not any(file.endswith(".safetensors") for file in files):
+ if any(file.endswith(_UNSIZED_WEIGHT_SUFFIXES) for file in files):
+ return None
+ continue
+ counts = _local_dit_parameter_counts(folder)
+ if not counts or not counts.keys() <= _FLOAT_DTYPES:
+ return None
+ sizes[name] = (
+ sum(counts.values()),
+ sum(n * _FLOAT_DTYPE_BYTES[dtype] for dtype, n in counts.items()),
+ )
+ return sizes
+ except Exception as exc:
+ logger.debug("Could not size the checkpoint of %s: %s", model_path, exc)
+ return None
+
+
def keep_checkpoint_mapped(*, weight_bytes: int, component: str) -> bool:
"""Whether a component's weights should stay on their file mapping.
diff --git a/python/sglang/multimodal_gen/runtime/server_args/auto_tune.py b/python/sglang/multimodal_gen/runtime/server_args/auto_tune.py
index cf012ceee5..f63e6a9860 100644
--- a/python/sglang/multimodal_gen/runtime/server_args/auto_tune.py
+++ b/python/sglang/multimodal_gen/runtime/server_args/auto_tune.py
@@ -18,6 +18,7 @@ from sglang.multimodal_gen.runtime.entrypoints.openai.realtime.registry import (
)
from sglang.multimodal_gen.runtime.loader.utils import (
BYTES_PER_GB,
+ checkpoint_component_sizes,
dit_parameter_count,
)
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload_components import (
@@ -26,12 +27,16 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload_co
LAYERWISE_OFFLOAD_IMAGE_ENCODER_GROUP,
LAYERWISE_OFFLOAD_TEXT_ENCODER_GROUP,
LAYERWISE_OFFLOAD_VAE_GROUP,
+ is_dit_component_name,
normalize_layerwise_offload_components,
)
from sglang.multimodal_gen.runtime.platforms import current_platform
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
from sglang.multimodal_gen.runtime.utils.model_overlay import resolve_model_overlay
-from sglang.multimodal_gen.runtime.utils.precision import resolve_precision
+from sglang.multimodal_gen.runtime.utils.precision import (
+ resolve_component_precision,
+ resolve_precision,
+)
if TYPE_CHECKING:
from sglang.multimodal_gen.runtime.server_args.server_args import ServerArgs
@@ -40,6 +45,8 @@ logger = init_logger(__name__)
# Picked, not measured; what stays free holds activations and the other components.
AUTO_DIT_MAX_DEVICE_FRACTION = 0.8
+# auto keeps every component resident once the least-free GPU holds all weights this many times
+AUTO_ALL_RESIDENT_WEIGHT_MULTIPLE = 2
def _uses_svdquant(nunchaku_config) -> bool:
@@ -110,6 +117,12 @@ DEFAULT_LAYERWISE_COMPONENT_ARG_NAMES = (
# larger so it only stays resident on very-high-memory gpus
IMAGE_GEN_KEEP_RESIDENT_MIN_AVAILABLE_GB = 45.0
DEFAULT_KEEP_RESIDENT_MIN_AVAILABLE_GB = 120.0
+ALL_OFFLOAD_COMPONENTS = (
+ LAYERWISE_OFFLOAD_DIT_GROUP,
+ LAYERWISE_OFFLOAD_TEXT_ENCODER_GROUP,
+ LAYERWISE_OFFLOAD_IMAGE_ENCODER_GROUP,
+ LAYERWISE_OFFLOAD_VAE_GROUP,
+)
# torch's CPU allocator (mimalloc since 2.13) keeps freed pages in its arenas
@@ -141,6 +154,8 @@ class ServerArgsAutoTuner:
self.server_args = server_args
self._explicit_dit_residency = self._has_explicit_dit_residency()
self._dit_fit_checked = False
+ # memory-rich tier decision, resolved on first use
+ self._all_resident: bool | None = None
def _deployment_config(self) -> ModelDeploymentConfig:
return self.server_args.pipeline_config.get_model_deployment_config()
@@ -242,13 +257,18 @@ class ServerArgsAutoTuner:
disable_threshold_gb = self._resolve_keep_resident_min_available_gb(
deployment_config
)
- if (
+ all_resident = self._keeps_all_components_resident()
+ if all_resident or (
min_available_gb is not None
and disable_threshold_gb is not None
and min_available_gb >= disable_threshold_gb
):
changed = []
- components = set(deployment_config.keep_resident_components)
+ components = set(
+ ALL_OFFLOAD_COMPONENTS
+ if all_resident
+ else deployment_config.keep_resident_components
+ )
if args.pipeline_config.task_type.is_image_gen():
components.add(LAYERWISE_OFFLOAD_DIT_GROUP)
if (
@@ -713,6 +733,8 @@ class ServerArgsAutoTuner:
args = self.server_args
if args.performance_mode != "auto" or current_platform.is_cpu():
return components
+ if self._keeps_all_components_resident():
+ return []
deployment_config = self._deployment_config()
threshold_gb = self._resolve_keep_resident_min_available_gb(deployment_config)
@@ -749,6 +771,7 @@ class ServerArgsAutoTuner:
or envs.SGLANG_CACHE_DIT_ENABLED
or args.use_fsdp_inference
or args.explicit_residency_mode("transformer") is not None
+ or self._keeps_all_components_resident()
):
return False
@@ -757,6 +780,66 @@ class ServerArgsAutoTuner:
return True
return self._dit_overflows_device_in_auto(deployment_config)
+ def _keeps_all_components_resident(self) -> bool:
+ if self._all_resident is None:
+ self._all_resident = self._fits_all_components_resident()
+ return self._all_resident
+
+ def _fits_all_components_resident(self) -> bool:
+ """Memory-rich auto tier: the least-free GPU holds every weight twice over.
+
+ Sized from the local checkpoint, so overrides, quantization and weights not
+ yet downloaded keep the per-model policy; one request's activations are
+ assumed, so batching and shared host/device pools are excluded too.
+ """
+ args = self.server_args
+ if (
+ args.performance_mode != "auto"
+ or current_platform.is_cpu()
+ or current_platform.is_mps()
+ or current_platform.device_shares_host_memory()
+ or args.batching_max_size > 1
+ or args.quantization is not None
+ or args.transformer_weights_path is not None
+ or _uses_svdquant(args.nunchaku_config)
+ or args.component_paths
+ or args.component_weights_paths
+ or args.component_quantizations
+ or resolve_model_overlay(args.model_path) is not None
+ ):
+ return False
+ available_gb = self._get_min_available_device_memory_gb()
+ if available_gb is None:
+ return False
+ sizes = checkpoint_component_sizes(
+ args.model_path, subfolder=args.model_subfolder, revision=args.revision
+ )
+ if sizes is None:
+ return False
+ weight_bytes = 0
+ for name, (num_params, stored_bytes) in sizes.items():
+ # loaded in the component's configured precision, else as stored
+ dtype = resolve_component_precision(args, name)
+ nbytes = num_params * dtype.itemsize if dtype is not None else stored_bytes
+ # tensor parallelism shards the DiT's linear layers across the ranks
+ weight_bytes += (
+ nbytes / (args.tp_size or 1) if is_dit_component_name(name) else nbytes
+ )
+ weight_gb = weight_bytes / BYTES_PER_GB
+ fits = available_gb >= AUTO_ALL_RESIDENT_WEIGHT_MULTIPLE * weight_gb
+ if fits:
+ logger.info(
+ "Auto memory policy for %s: %.1f GiB free on the least-free "
+ "selected GPU holds the %.1f GiB of weights per GPU at least "
+ "%dx, so every component without an explicit placement stays "
+ "resident.",
+ args.pipeline_config.__class__.__name__,
+ available_gb,
+ weight_gb,
+ AUTO_ALL_RESIDENT_WEIGHT_MULTIPLE,
+ )
+ return fits
+
def _dit_overflows_device_in_auto(
self, deployment_config: ModelDeploymentConfig
) -> bool:
diff --git a/python/sglang/multimodal_gen/test/server/gpu_cases.py b/python/sglang/multimodal_gen/test/server/gpu_cases.py
index 97de0ec4cd..41ec437f94 100644
--- a/python/sglang/multimodal_gen/test/server/gpu_cases.py
+++ b/python/sglang/multimodal_gen/test/server/gpu_cases.py
@@ -326,7 +326,8 @@ ONE_GPU_CASES: list[DiffusionTestCase] = [
# request the stage regains that whole transfer, well past the
# non_denoise_stage tolerance. load_peak_vram carries the placed set, so
# it also pins that the placement happens at load, not on the first use.
- # The encoder is already layerwise under the default component set.
+ # The encoder is pinned layerwise: on an 80 GB card auto would keep the
+ # whole 1.3B checkpoint resident.
# No consistency check: the lifetime moves weights, not math, so the
# output is the base case's and that case already guards it.
DiffusionTestCase(
@@ -334,6 +335,7 @@ ONE_GPU_CASES: list[DiffusionTestCase] = [
DiffusionServerArgs(
model_path=DEFAULT_WAN_2_1_T2V_1_3B_MODEL_NAME_FOR_TEST,
extras=[
+ "--component-residency text_encoder=layerwise-offload "
"--layerwise-resident-layers text_encoder=0.8 "
"--layerwise-residency-lifetime text_encoder=permanent",
],
diff --git a/python/sglang/multimodal_gen/test/server/perf_baselines/b200.json b/python/sglang/multimodal_gen/test/server/perf_baselines/b200.json
index 8b41fd60fb..46afb666f2 100644
--- a/python/sglang/multimodal_gen/test/server/perf_baselines/b200.json
+++ b/python/sglang/multimodal_gen/test/server/perf_baselines/b200.json
@@ -272,10 +272,10 @@
"expected_avg_denoise_ms": 478.63,
"expected_median_denoise_ms": 443.42,
"estimated_full_test_time_s": 181.8,
- "load_peak_vram_mb": 4920.72,
- "runtime_peak_vram_mb": 9895.98,
- "load_peak_allocated_mb": 4858.5,
- "runtime_peak_allocated_mb": 6460.38
+ "load_peak_vram_mb": 25490.0,
+ "runtime_peak_vram_mb": 35568.0,
+ "load_peak_allocated_mb": 24622.71,
+ "runtime_peak_allocated_mb": 28881.55
},
"ideogram4_nvfp4_t2i": {
"expected_load_ms": 33439.28,
diff --git a/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json b/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json
index 72208d860e..a1420df63f 100644
--- a/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json
+++ b/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json
@@ -465,10 +465,10 @@
"expected_e2e_ms": 6693.01,
"expected_avg_denoise_ms": 128.85,
"expected_median_denoise_ms": 130.22,
- "load_peak_vram_mb": 23594.0,
- "runtime_peak_vram_mb": 28540.0,
- "load_peak_allocated_mb": 23572.0,
- "runtime_peak_allocated_mb": 25667.0,
+ "load_peak_vram_mb": 32296.0,
+ "runtime_peak_vram_mb": 36766.0,
+ "load_peak_allocated_mb": 32191.79,
+ "runtime_peak_allocated_mb": 34669.69,
"estimated_full_test_time_s": 80
},
"flux_2_image_t2i": {
@@ -850,10 +850,10 @@
"expected_e2e_ms": 4244.27,
"expected_avg_denoise_ms": 79.98,
"expected_median_denoise_ms": 81.01,
- "load_peak_vram_mb": 23284.0,
- "runtime_peak_vram_mb": 28148.0,
- "load_peak_allocated_mb": 23262.0,
- "runtime_peak_allocated_mb": 25541.0,
+ "load_peak_vram_mb": 27722.0,
+ "runtime_peak_vram_mb": 32332.0,
+ "load_peak_allocated_mb": 27651.29,
+ "runtime_peak_allocated_mb": 30128.94,
"estimated_full_test_time_s": 50.0
},
"zimage_image_t2i": {
@@ -880,10 +880,10 @@
"expected_e2e_ms": 906.14,
"expected_avg_denoise_ms": 82.61,
"expected_median_denoise_ms": 90.0,
- "load_peak_vram_mb": 13120.0,
- "runtime_peak_vram_mb": 17776.0,
- "load_peak_allocated_mb": 12852.0,
- "runtime_peak_allocated_mb": 15133.0,
+ "load_peak_vram_mb": 20546.0,
+ "runtime_peak_vram_mb": 24492.0,
+ "load_peak_allocated_mb": 20217.47,
+ "runtime_peak_allocated_mb": 22117.63,
"estimated_full_test_time_s": 50
},
"zimage_image_t2i_fp8": {
@@ -940,10 +940,10 @@
"expected_e2e_ms": 910.85,
"expected_avg_denoise_ms": 83.08,
"expected_median_denoise_ms": 89.93,
- "load_peak_vram_mb": 14196.0,
- "runtime_peak_vram_mb": 17612.0,
- "load_peak_allocated_mb": 13059.0,
- "runtime_peak_allocated_mb": 15307.0,
+ "load_peak_vram_mb": 21210.0,
+ "runtime_peak_vram_mb": 24626.0,
+ "load_peak_allocated_mb": 20453.97,
+ "runtime_peak_allocated_mb": 22291.56,
"estimated_full_test_time_s": 90
},
"zimage_image_t2i_2_gpus": {
@@ -970,10 +970,10 @@
"expected_e2e_ms": 752.84,
"expected_avg_denoise_ms": 51.13,
"expected_median_denoise_ms": 54.3,
- "load_peak_vram_mb": 13120.0,
- "runtime_peak_vram_mb": 17602.0,
- "load_peak_allocated_mb": 12852.0,
- "runtime_peak_allocated_mb": 15131.0,
+ "load_peak_vram_mb": 20546.0,
+ "runtime_peak_vram_mb": 24340.0,
+ "load_peak_allocated_mb": 20217.47,
+ "runtime_peak_allocated_mb": 22115.25,
"estimated_full_test_time_s": 50.0
},
"qwen_image_edit_ti2i": {
@@ -1246,10 +1246,10 @@
"expected_e2e_ms": 4739.06,
"expected_avg_denoise_ms": 76.99,
"expected_median_denoise_ms": 98.28,
- "load_peak_vram_mb": 7814.0,
- "runtime_peak_vram_mb": 18078.0,
- "load_peak_allocated_mb": 3641.0,
- "runtime_peak_allocated_mb": 9730.0,
+ "load_peak_vram_mb": 26126.0,
+ "runtime_peak_vram_mb": 33340.0,
+ "load_peak_allocated_mb": 25740.95,
+ "runtime_peak_allocated_mb": 31408.67,
"estimated_full_test_time_s": 60
},
"wan2_1_t2v_1.3b_encoder_permanent_residents": {
@@ -1389,10 +1389,10 @@
"expected_e2e_ms": 8106.87,
"expected_avg_denoise_ms": 143.82,
"expected_median_denoise_ms": 144.57,
- "load_peak_vram_mb": 7814.0,
- "runtime_peak_vram_mb": 18078.0,
- "load_peak_allocated_mb": 3641.0,
- "runtime_peak_allocated_mb": 9673.0,
+ "load_peak_vram_mb": 26126.0,
+ "runtime_peak_vram_mb": 33340.0,
+ "load_peak_allocated_mb": 25740.95,
+ "runtime_peak_allocated_mb": 31354.62,
"estimated_full_test_time_s": 150
},
"wan2_1_t2v_1.3b_cfg_parallel": {
@@ -1460,10 +1460,10 @@
"expected_e2e_ms": 4435.63,
"expected_avg_denoise_ms": 73.51,
"expected_median_denoise_ms": 74.05,
- "load_peak_vram_mb": 5422.0,
- "runtime_peak_vram_mb": 11972.0,
- "load_peak_allocated_mb": 5274.0,
- "runtime_peak_allocated_mb": 9251.0,
+ "load_peak_vram_mb": 15292.0,
+ "runtime_peak_vram_mb": 19728.0,
+ "load_peak_allocated_mb": 14906.86,
+ "runtime_peak_allocated_mb": 18094.7,
"estimated_full_test_time_s": 60.0
},
"turbo_wan2_1_t2v_1.3b": {
@@ -1629,10 +1629,10 @@
"expected_e2e_ms": 15599.56,
"expected_avg_denoise_ms": 253.29,
"expected_median_denoise_ms": 256.21,
- "load_peak_vram_mb": 17610.0,
- "runtime_peak_vram_mb": 39856.0,
- "load_peak_allocated_mb": 11629.0,
- "runtime_peak_allocated_mb": 26066.0,
+ "load_peak_vram_mb": 36644.0,
+ "runtime_peak_vram_mb": 53216.0,
+ "load_peak_allocated_mb": 36480.46,
+ "runtime_peak_allocated_mb": 49073.06,
"estimated_full_test_time_s": 80
},
"qwen_image_edit_2509_ti2i": {
@@ -1784,10 +1784,10 @@
"expected_e2e_ms": 3073.51,
"expected_avg_denoise_ms": 83.86,
"expected_median_denoise_ms": 112.67,
- "load_peak_vram_mb": 17610.0,
- "runtime_peak_vram_mb": 39054.0,
- "load_peak_allocated_mb": 11629.0,
- "runtime_peak_allocated_mb": 26051.0,
+ "load_peak_vram_mb": 36644.0,
+ "runtime_peak_vram_mb": 53636.0,
+ "load_peak_allocated_mb": 36480.46,
+ "runtime_peak_allocated_mb": 49057.2,
"estimated_full_test_time_s": 60
},
"fast_hunyuan_video": {
@@ -1812,11 +1812,11 @@
"expected_e2e_ms": 9123.68,
"expected_avg_denoise_ms": 871.39,
"expected_median_denoise_ms": 1010.9,
- "load_peak_vram_mb": 27320.0,
- "runtime_peak_vram_mb": 35926.0,
+ "load_peak_vram_mb": 40050.0,
+ "runtime_peak_vram_mb": 47480.0,
"warmup_peak_vram_mb": 56748.0,
- "load_peak_allocated_mb": 26241.0,
- "runtime_peak_allocated_mb": 31922.0,
+ "load_peak_allocated_mb": 39773.7,
+ "runtime_peak_allocated_mb": 44331.45,
"estimated_full_test_time_s": 80
},
"wan2_2_i2v_a14b_2gpu": {
@@ -2287,10 +2287,10 @@
"expected_e2e_ms": 8138.0,
"expected_avg_denoise_ms": 146.48,
"expected_median_denoise_ms": 147.68,
- "load_peak_vram_mb": 7814.0,
- "runtime_peak_vram_mb": 18350.0,
- "load_peak_allocated_mb": 3641.0,
- "runtime_peak_allocated_mb": 9999.0,
+ "load_peak_vram_mb": 26126.0,
+ "runtime_peak_vram_mb": 33442.0,
+ "load_peak_allocated_mb": 25740.95,
+ "runtime_peak_allocated_mb": 31666.73,
"estimated_full_test_time_s": 90
},
"wan2_1_t2v_1_3b_dynamic_lora_default_quality_1gpu": {
@@ -2557,10 +2557,10 @@
"expected_e2e_ms": 1042.43,
"expected_avg_denoise_ms": 82.26,
"expected_median_denoise_ms": 81.75,
- "load_peak_vram_mb": 7176.0,
- "runtime_peak_vram_mb": 12634.0,
- "load_peak_allocated_mb": 7033.0,
- "runtime_peak_allocated_mb": 9311.0,
+ "load_peak_vram_mb": 14558.0,
+ "runtime_peak_vram_mb": 19230.0,
+ "load_peak_allocated_mb": 14399.24,
+ "runtime_peak_allocated_mb": 16295.65,
"estimated_full_test_time_s": 40.0
},
"hunyuan3d_shape_gen": {
@@ -2700,10 +2700,10 @@
"expected_e2e_ms": 8273.83,
"expected_avg_denoise_ms": 146.61,
"expected_median_denoise_ms": 147.3,
- "load_peak_vram_mb": 7814.0,
- "runtime_peak_vram_mb": 18080.0,
- "load_peak_allocated_mb": 3641.0,
- "runtime_peak_allocated_mb": 9673.0,
+ "load_peak_vram_mb": 26126.0,
+ "runtime_peak_vram_mb": 34026.0,
+ "load_peak_allocated_mb": 25740.95,
+ "runtime_peak_allocated_mb": 31340.91,
"estimated_full_test_time_s": 70
},
"flux_2_image_t2i_upscaling_4x": {
@@ -2843,10 +2843,10 @@
"expected_e2e_ms": 8333.66,
"expected_avg_denoise_ms": 144.68,
"expected_median_denoise_ms": 144.74,
- "load_peak_vram_mb": 7814.0,
- "runtime_peak_vram_mb": 24380.0,
- "load_peak_allocated_mb": 3641.0,
- "runtime_peak_allocated_mb": 10508.0,
+ "load_peak_vram_mb": 26126.0,
+ "runtime_peak_vram_mb": 39714.0,
+ "load_peak_allocated_mb": 25740.95,
+ "runtime_peak_allocated_mb": 31340.91,
"estimated_full_test_time_s": 100
},
"wan2_1_t2v_1.3b_frame_interp_2x_upscaling_4x": {
@@ -2914,10 +2914,10 @@
"expected_e2e_ms": 8099.03,
"expected_avg_denoise_ms": 143.71,
"expected_median_denoise_ms": 144.67,
- "load_peak_vram_mb": 7814.0,
- "runtime_peak_vram_mb": 24388.0,
- "load_peak_allocated_mb": 3641.0,
- "runtime_peak_allocated_mb": 10531.0,
+ "load_peak_vram_mb": 26126.0,
+ "runtime_peak_vram_mb": 40406.0,
+ "load_peak_allocated_mb": 25740.95,
+ "runtime_peak_allocated_mb": 31340.91,
"estimated_full_test_time_s": 100
},
"ltx_2.3_one_stage_ti2v": {
diff --git a/python/sglang/multimodal_gen/test/unit/conftest.py b/python/sglang/multimodal_gen/test/unit/conftest.py
index 8f68d0dd9d..53f3f52404 100644
--- a/python/sglang/multimodal_gen/test/unit/conftest.py
+++ b/python/sglang/multimodal_gen/test/unit/conftest.py
@@ -127,6 +127,11 @@ def mock_cuda_platform(
"sglang.multimodal_gen.runtime.platforms.current_platform.enable_dit_layerwise_offload_by_default",
return_value=True,
),
+ # checkpoint sizing reads the local HF cache; tests that need it pass a size
+ patch(
+ "sglang.multimodal_gen.runtime.server_args.auto_tune.checkpoint_component_sizes",
+ return_value=None,
+ ),
):
yield
diff --git a/python/sglang/multimodal_gen/test/unit/test_server_args.py b/python/sglang/multimodal_gen/test/unit/test_server_args.py
index 506f29aa18..fc50e7f6a3 100644
--- a/python/sglang/multimodal_gen/test/unit/test_server_args.py
+++ b/python/sglang/multimodal_gen/test/unit/test_server_args.py
@@ -1232,6 +1232,8 @@ class TestOffloadDefaults(_CudaPlatformTestCase):
memory_gb=80,
available_memory_gb=None,
dit_parameter_count=None,
+ checkpoint_sizes=None,
+ is_cuda=True,
kwargs=None,
):
def get_available_gpu_memory(device_id=0, **_kwargs):
@@ -1251,6 +1253,10 @@ class TestOffloadDefaults(_CudaPlatformTestCase):
else Mock(return_value=dit_parameter_count)
),
),
+ patch(
+ "sglang.multimodal_gen.runtime.server_args.auto_tune.checkpoint_component_sizes",
+ return_value=checkpoint_sizes,
+ ),
patch(
"sglang.multimodal_gen.runtime.platforms.current_platform.is_cpu",
return_value=False,
@@ -1261,7 +1267,7 @@ class TestOffloadDefaults(_CudaPlatformTestCase):
),
patch(
"sglang.multimodal_gen.runtime.platforms.current_platform.is_cuda",
- return_value=True,
+ return_value=is_cuda,
),
patch(
"sglang.multimodal_gen.runtime.platforms.current_platform.enable_dit_layerwise_offload_by_default",
@@ -2107,6 +2113,101 @@ class TestOffloadDefaults(_CudaPlatformTestCase):
["text_encoder", "image_encoder"],
)
+ def _zimage_turbo(
+ self, *, memory_gb=141, available_memory_gb=139, checkpoint_sizes=None, **kwargs
+ ):
+ return self._from_dict_with_pipeline_config(
+ ZImagePipelineConfig(),
+ memory_gb=memory_gb,
+ available_memory_gb=available_memory_gb,
+ checkpoint_sizes=checkpoint_sizes,
+ kwargs={
+ "model_path": "Tongyi-MAI/Z-Image-Turbo",
+ "performance_mode": "auto",
+ **kwargs,
+ },
+ )
+
+ # (parameters, stored bytes) of Tongyi-MAI/Z-Image-Turbo: the DiT is stored in
+ # fp32 (22.9 GiB) and served in bf16, 19.3 GiB per GPU in all
+ ZIMAGE_TURBO_SIZES = {
+ "transformer": (6_154_908_736, 24_619_634_944),
+ "text_encoder": (4_022_468_096, 8_044_936_192),
+ "vae": (83_819_683, 335_278_732),
+ }
+
+ def _residency(self, args):
+ return {
+ name: args.residency_mode(name)
+ for name in ("transformer", "text_encoder", "image_encoder", "vae")
+ }
+
+ def test_auto_keeps_every_component_resident_when_weights_fit_twice(self):
+ args = self._zimage_turbo(checkpoint_sizes=self.ZIMAGE_TURBO_SIZES)
+
+ self.assertIsNone(args.layerwise_offload_components)
+ self.assertFalse(args.dit_cpu_offload)
+ self.assertFalse(args.text_encoder_cpu_offload)
+ self.assertEqual(set(self._residency(args).values()), {RESIDENT})
+
+ def test_auto_tier_sizes_weights_in_their_serving_precision(self):
+ # fits twice in 45 GiB at bf16; the fp32-stored bytes would not
+ args = self._zimage_turbo(
+ memory_gb=48,
+ available_memory_gb=45,
+ checkpoint_sizes=self.ZIMAGE_TURBO_SIZES,
+ )
+ self.assertEqual(set(self._residency(args).values()), {RESIDENT})
+
+ def test_auto_tier_leaves_consumer_gpus_on_their_policy(self):
+ for memory_gb, available_gb in ((32, 31), (24, 23)):
+ with self.subTest(memory_gb=memory_gb):
+ sized = self._zimage_turbo(
+ memory_gb=memory_gb,
+ available_memory_gb=available_gb,
+ checkpoint_sizes=self.ZIMAGE_TURBO_SIZES,
+ )
+ unsized = self._zimage_turbo(
+ memory_gb=memory_gb, available_memory_gb=available_gb
+ )
+ self.assertEqual(self._residency(sized), self._residency(unsized))
+ self.assertNotEqual(sized.residency_mode("text_encoder"), RESIDENT)
+
+ def test_auto_tier_skips_batching_shared_memory_and_unsized_checkpoints(self):
+ unsized = self._zimage_turbo()
+ self.assertEqual(unsized.residency_mode("text_encoder"), LAYERWISE_OFFLOAD)
+ batching = self._zimage_turbo(
+ checkpoint_sizes=self.ZIMAGE_TURBO_SIZES, batching_max_size=2
+ )
+ self.assertEqual(batching.residency_mode("text_encoder"), LAYERWISE_OFFLOAD)
+ with patch.object(
+ current_platform, "device_shares_host_memory", return_value=True
+ ):
+ shared = self._zimage_turbo(checkpoint_sizes=self.ZIMAGE_TURBO_SIZES)
+ self.assertEqual(shared.residency_mode("text_encoder"), LAYERWISE_OFFLOAD)
+
+ def test_auto_tier_replaces_whole_component_offload_off_cuda(self):
+ # off CUDA auto has no layerwise defaults and offloads the encoder whole
+ unsized = self._from_dict_with_pipeline_config(
+ ZImagePipelineConfig(),
+ memory_gb=192,
+ available_memory_gb=190,
+ is_cuda=False,
+ kwargs={"model_path": "Tongyi-MAI/Z-Image-Turbo"},
+ )
+ sized = self._from_dict_with_pipeline_config(
+ ZImagePipelineConfig(),
+ memory_gb=192,
+ available_memory_gb=190,
+ checkpoint_sizes=self.ZIMAGE_TURBO_SIZES,
+ is_cuda=False,
+ kwargs={"model_path": "Tongyi-MAI/Z-Image-Turbo"},
+ )
+
+ self.assertTrue(unsized.text_encoder_cpu_offload)
+ self.assertFalse(sized.text_encoder_cpu_offload)
+ self.assertEqual(set(self._residency(sized).values()), {RESIDENT})
+
def test_auto_lingbot_keeps_dit_resident_on_h100(self):
args = self._from_dict_with_pipeline_config(
LingBotWorldCausalDMDConfig(),
@@ -2215,6 +2316,45 @@ class TestOffloadDefaults(_CudaPlatformTestCase):
["dit", "text_encoder", "image_encoder", "vae"],
)
+ def test_auto_wan2_2_a14b_keeps_both_experts_resident_above_120gb(self):
+ for pipeline_config, model_path in (
+ (Wan2_2_T2V_A14B_Config(), "Wan-AI/Wan2.2-T2V-A14B-Diffusers"),
+ (Wan2_2_I2V_A14B_Config(), "Wan-AI/Wan2.2-I2V-A14B-Diffusers"),
+ ):
+ with self.subTest(pipeline_config=pipeline_config.__class__.__name__):
+ args = self._from_dict_with_pipeline_config(
+ pipeline_config,
+ memory_gb=141,
+ available_memory_gb=139,
+ kwargs={"model_path": model_path, "performance_mode": "auto"},
+ )
+
+ self.assertFalse(args.dit_cpu_offload)
+ self.assertEqual(args.residency_mode("transformer"), RESIDENT)
+ self.assertEqual(args.residency_mode("transformer_2"), RESIDENT)
+ self.assertEqual(args.residency_mode("vae"), RESIDENT)
+ self.assertEqual(
+ args.layerwise_offload_components,
+ ["text_encoder", "image_encoder"],
+ )
+
+ # two 14B experts plus T5 fit twice over: the encoder joins them
+ rich = self._from_dict_with_pipeline_config(
+ pipeline_config,
+ memory_gb=288,
+ available_memory_gb=280,
+ checkpoint_sizes={
+ "transformer": (14_288_901_184, 57_155_604_736),
+ "transformer_2": (14_288_901_184, 57_155_604_736),
+ "text_encoder": (5_680_910_336, 22_723_641_344),
+ "vae": (126_892_531, 507_570_124),
+ },
+ kwargs={"model_path": model_path, "performance_mode": "auto"},
+ )
+ self.assertIsNone(rich.layerwise_offload_components)
+ self.assertEqual(rich.residency_mode("text_encoder"), RESIDENT)
+ self.assertEqual(rich.residency_mode("transformer_2"), RESIDENT)
+
def test_auto_wan2_1_14b_keeps_dit_resident_on_h100(self):
for pipeline_config, model_path in (
(WanT2V720PConfig(), "Wan-AI/Wan2.1-T2V-14B-Diffusers"),
diff --git a/python/sglang/multimodal_gen/test/unit/test_weight_utils.py b/python/sglang/multimodal_gen/test/unit/test_weight_utils.py
index 3591f62848..a349e9dbba 100644
--- a/python/sglang/multimodal_gen/test/unit/test_weight_utils.py
+++ b/python/sglang/multimodal_gen/test/unit/test_weight_utils.py
@@ -2,6 +2,7 @@
"""Unit tests for multimodal weight-loading utilities."""
import os
+import shutil
import tempfile
import unittest
from pathlib import Path
@@ -15,6 +16,7 @@ from safetensors.torch import save_file as safetensors_save_file
from sglang.multimodal_gen.runtime.loader.utils import (
_list_safetensors_files,
checkpoint_bytes,
+ checkpoint_component_sizes,
dit_parameter_count,
)
from sglang.multimodal_gen.runtime.loader.weight_utils import (
@@ -266,6 +268,79 @@ class TestDitParameterCount(CustomTestCase):
self.assertEqual(hub_download.called, use_modelscope == "0")
+def _write_pipeline(root: Path) -> None:
+ """A Diffusers layout: a 58-byte DiT, a 16-byte encoder and weightless parts."""
+ _write_sharded_transformer(root)
+ (root / "text_encoder").mkdir()
+ safetensors_save_file(
+ {"e": torch.zeros(4)}, root / "text_encoder" / "model.safetensors"
+ )
+ for weightless in ("tokenizer", "scheduler"):
+ (root / weightless).mkdir()
+ (root / weightless / "config.json").write_text("{}")
+ (root / "model_index.json").write_text(
+ '{"_class_name": "P", "transformer": ["diffusers", "T"], '
+ '"text_encoder": ["transformers", "E"], "tokenizer": ["transformers", "K"], '
+ '"scheduler": ["diffusers", "S"], "safety_checker": [null, null], '
+ '"boundary_ratio": 0.9}'
+ )
+
+
+class TestCheckpointComponentSizes(CustomTestCase):
+ SIZES = {"transformer": (17, 58), "text_encoder": (4, 16)}
+
+ def test_sizes_every_weighted_component(self):
+ with tempfile.TemporaryDirectory() as tmpdir:
+ _write_pipeline(Path(tmpdir) / "sub")
+
+ self.assertEqual(
+ checkpoint_component_sizes(tmpdir, subfolder="sub", revision=None),
+ self.SIZES,
+ )
+
+ def test_cached_hub_snapshot_is_sized_without_the_network(self):
+ with tempfile.TemporaryDirectory() as tmpdir:
+ _write_pipeline(Path(tmpdir))
+ with patch(_SNAPSHOT_DOWNLOAD, return_value=tmpdir) as snapshot_download:
+ self.assertEqual(
+ checkpoint_component_sizes(
+ "org/model", subfolder=None, revision="abc"
+ ),
+ self.SIZES,
+ )
+ self.assertTrue(snapshot_download.call_args.kwargs["local_files_only"])
+
+ def test_partial_download_other_formats_or_quantized_weights_are_unsized(self):
+ cases = {
+ # only model_index.json fetched so far, as before the worker downloads
+ "index only": lambda root: shutil.rmtree(root / "text_encoder"),
+ "missing shard": lambda root: (
+ root / "transformer" / "shard-2.safetensors"
+ ).unlink(),
+ "bin weights": lambda root: (
+ (root / "text_encoder" / "model.safetensors").unlink(),
+ (root / "text_encoder" / "pytorch_model.bin").write_bytes(b"0"),
+ ),
+ "quantized": lambda root: safetensors_save_file(
+ {"q": torch.zeros(4, dtype=torch.float8_e4m3fn)},
+ root / "text_encoder" / "model.safetensors",
+ ),
+ }
+ for name, damage in cases.items():
+ with self.subTest(name), tempfile.TemporaryDirectory() as tmpdir:
+ _write_pipeline(Path(tmpdir))
+ damage(Path(tmpdir))
+ self.assertIsNone(
+ checkpoint_component_sizes(tmpdir, subfolder=None, revision=None)
+ )
+
+ def test_uncached_repo_is_unsized(self):
+ with patch(_SNAPSHOT_DOWNLOAD, side_effect=LocalEntryNotFoundError("miss")):
+ self.assertIsNone(
+ checkpoint_component_sizes("org/model", subfolder=None, revision=None)
+ )
+
+
class TestDisableRunaiStreamerRankDiscoveryCollective(unittest.TestCase):
def test_never_touches_torch_distributed_even_when_initialized(self):
from runai_model_streamer.distributed_streamer.distributed_streamer import (