[Perf] diffusion: Keep every component resident in auto mode when memory allows (#45978)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Mick Qian
Claude Opus 5.5
parent
be8600d02d
commit
4e2d5b17ad
@@ -288,9 +288,12 @@ class Wan2_2_T2V_A14B_Config(WanT2V480PConfig):
|
||||
self.dit_config.torch_compile_mode = "default"
|
||||
|
||||
def get_model_deployment_config(self) -> ModelDeploymentConfig:
|
||||
# both 14B experts plus T5 take ~66 GB per GPU; 120 keeps 80 GB cards streaming
|
||||
return ModelDeploymentConfig(
|
||||
dit_layerwise_offload_modes=("auto", "memory"),
|
||||
auto_dit_offload_prefetch_size=2,
|
||||
keep_resident_min_available_gb=120,
|
||||
keep_resident_components=("dit", "vae"),
|
||||
)
|
||||
|
||||
|
||||
@@ -305,9 +308,12 @@ class Wan2_2_I2V_A14B_Config(WanI2V720PConfig):
|
||||
self.dit_config.boundary_ratio = self.boundary_ratio
|
||||
|
||||
def get_model_deployment_config(self) -> ModelDeploymentConfig:
|
||||
# both 14B experts plus T5 take ~66 GB per GPU; 120 keeps 80 GB cards streaming
|
||||
return ModelDeploymentConfig(
|
||||
dit_layerwise_offload_modes=("auto", "memory"),
|
||||
auto_dit_offload_prefetch_size=2,
|
||||
keep_resident_min_available_gb=120,
|
||||
keep_resident_components=("dit", "vae"),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -419,7 +419,9 @@ def checkpoint_bytes(model_path: str) -> int:
|
||||
|
||||
_DIT_SUBFOLDER = "transformer"
|
||||
_DIT_SINGLE_FILE = "diffusion_pytorch_model.safetensors"
|
||||
_FLOAT_DTYPES = frozenset({"F64", "F32", "F16", "BF16"})
|
||||
_FLOAT_DTYPE_BYTES = {"F64": 8, "F32": 4, "F16": 2, "BF16": 2}
|
||||
_FLOAT_DTYPES = frozenset(_FLOAT_DTYPE_BYTES)
|
||||
_UNSIZED_WEIGHT_SUFFIXES = (".bin", ".pt", ".pth", ".ckpt", ".gguf")
|
||||
|
||||
|
||||
def _index_shard_names(index_path: str) -> list[str]:
|
||||
@@ -507,6 +509,59 @@ def dit_parameter_count(
|
||||
return sum(counts.values())
|
||||
|
||||
|
||||
def checkpoint_component_sizes(
|
||||
model_path: str, *, subfolder: str | None, revision: str | None
|
||||
) -> dict[str, tuple[int, int]] | None:
|
||||
"""(parameter count, stored bytes) of each model_index.json component of a
|
||||
locally cached Diffusers checkpoint, from safetensors headers.
|
||||
|
||||
Weightless components (tokenizers, schedulers) are left out. None when any
|
||||
component cannot be sized from local float safetensors (not cached, missing
|
||||
shards, other formats, quantized weights).
|
||||
"""
|
||||
try:
|
||||
model_path = os.path.expanduser(model_path)
|
||||
if os.path.isdir(model_path):
|
||||
root = model_path
|
||||
elif os.path.exists(model_path) or envs.SGLANG_USE_MODELSCOPE.get():
|
||||
return None
|
||||
else:
|
||||
root = snapshot_download(
|
||||
repo_id=model_path,
|
||||
revision=revision,
|
||||
allow_patterns=["*model_index.json"],
|
||||
local_files_only=True,
|
||||
)
|
||||
if subfolder:
|
||||
root = os.path.join(root, subfolder)
|
||||
with open(os.path.join(root, "model_index.json")) as f:
|
||||
components = [
|
||||
name
|
||||
for name, spec in json.load(f).items()
|
||||
if isinstance(spec, list) and spec[0] is not None
|
||||
]
|
||||
sizes = {}
|
||||
for name in components:
|
||||
folder = os.path.join(root, name)
|
||||
# a component folder not yet downloaded raises here
|
||||
files = os.listdir(folder)
|
||||
if not any(file.endswith(".safetensors") for file in files):
|
||||
if any(file.endswith(_UNSIZED_WEIGHT_SUFFIXES) for file in files):
|
||||
return None
|
||||
continue
|
||||
counts = _local_dit_parameter_counts(folder)
|
||||
if not counts or not counts.keys() <= _FLOAT_DTYPES:
|
||||
return None
|
||||
sizes[name] = (
|
||||
sum(counts.values()),
|
||||
sum(n * _FLOAT_DTYPE_BYTES[dtype] for dtype, n in counts.items()),
|
||||
)
|
||||
return sizes
|
||||
except Exception as exc:
|
||||
logger.debug("Could not size the checkpoint of %s: %s", model_path, exc)
|
||||
return None
|
||||
|
||||
|
||||
def keep_checkpoint_mapped(*, weight_bytes: int, component: str) -> bool:
|
||||
"""Whether a component's weights should stay on their file mapping.
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ from sglang.multimodal_gen.runtime.entrypoints.openai.realtime.registry import (
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.loader.utils import (
|
||||
BYTES_PER_GB,
|
||||
checkpoint_component_sizes,
|
||||
dit_parameter_count,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload_components import (
|
||||
@@ -26,12 +27,16 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload_co
|
||||
LAYERWISE_OFFLOAD_IMAGE_ENCODER_GROUP,
|
||||
LAYERWISE_OFFLOAD_TEXT_ENCODER_GROUP,
|
||||
LAYERWISE_OFFLOAD_VAE_GROUP,
|
||||
is_dit_component_name,
|
||||
normalize_layerwise_offload_components,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.platforms import current_platform
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
from sglang.multimodal_gen.runtime.utils.model_overlay import resolve_model_overlay
|
||||
from sglang.multimodal_gen.runtime.utils.precision import resolve_precision
|
||||
from sglang.multimodal_gen.runtime.utils.precision import (
|
||||
resolve_component_precision,
|
||||
resolve_precision,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.multimodal_gen.runtime.server_args.server_args import ServerArgs
|
||||
@@ -40,6 +45,8 @@ logger = init_logger(__name__)
|
||||
|
||||
# Picked, not measured; what stays free holds activations and the other components.
|
||||
AUTO_DIT_MAX_DEVICE_FRACTION = 0.8
|
||||
# auto keeps every component resident once the least-free GPU holds all weights this many times
|
||||
AUTO_ALL_RESIDENT_WEIGHT_MULTIPLE = 2
|
||||
|
||||
|
||||
def _uses_svdquant(nunchaku_config) -> bool:
|
||||
@@ -110,6 +117,12 @@ DEFAULT_LAYERWISE_COMPONENT_ARG_NAMES = (
|
||||
# larger so it only stays resident on very-high-memory gpus
|
||||
IMAGE_GEN_KEEP_RESIDENT_MIN_AVAILABLE_GB = 45.0
|
||||
DEFAULT_KEEP_RESIDENT_MIN_AVAILABLE_GB = 120.0
|
||||
ALL_OFFLOAD_COMPONENTS = (
|
||||
LAYERWISE_OFFLOAD_DIT_GROUP,
|
||||
LAYERWISE_OFFLOAD_TEXT_ENCODER_GROUP,
|
||||
LAYERWISE_OFFLOAD_IMAGE_ENCODER_GROUP,
|
||||
LAYERWISE_OFFLOAD_VAE_GROUP,
|
||||
)
|
||||
|
||||
|
||||
# torch's CPU allocator (mimalloc since 2.13) keeps freed pages in its arenas
|
||||
@@ -141,6 +154,8 @@ class ServerArgsAutoTuner:
|
||||
self.server_args = server_args
|
||||
self._explicit_dit_residency = self._has_explicit_dit_residency()
|
||||
self._dit_fit_checked = False
|
||||
# memory-rich tier decision, resolved on first use
|
||||
self._all_resident: bool | None = None
|
||||
|
||||
def _deployment_config(self) -> ModelDeploymentConfig:
|
||||
return self.server_args.pipeline_config.get_model_deployment_config()
|
||||
@@ -242,13 +257,18 @@ class ServerArgsAutoTuner:
|
||||
disable_threshold_gb = self._resolve_keep_resident_min_available_gb(
|
||||
deployment_config
|
||||
)
|
||||
if (
|
||||
all_resident = self._keeps_all_components_resident()
|
||||
if all_resident or (
|
||||
min_available_gb is not None
|
||||
and disable_threshold_gb is not None
|
||||
and min_available_gb >= disable_threshold_gb
|
||||
):
|
||||
changed = []
|
||||
components = set(deployment_config.keep_resident_components)
|
||||
components = set(
|
||||
ALL_OFFLOAD_COMPONENTS
|
||||
if all_resident
|
||||
else deployment_config.keep_resident_components
|
||||
)
|
||||
if args.pipeline_config.task_type.is_image_gen():
|
||||
components.add(LAYERWISE_OFFLOAD_DIT_GROUP)
|
||||
if (
|
||||
@@ -713,6 +733,8 @@ class ServerArgsAutoTuner:
|
||||
args = self.server_args
|
||||
if args.performance_mode != "auto" or current_platform.is_cpu():
|
||||
return components
|
||||
if self._keeps_all_components_resident():
|
||||
return []
|
||||
|
||||
deployment_config = self._deployment_config()
|
||||
threshold_gb = self._resolve_keep_resident_min_available_gb(deployment_config)
|
||||
@@ -749,6 +771,7 @@ class ServerArgsAutoTuner:
|
||||
or envs.SGLANG_CACHE_DIT_ENABLED
|
||||
or args.use_fsdp_inference
|
||||
or args.explicit_residency_mode("transformer") is not None
|
||||
or self._keeps_all_components_resident()
|
||||
):
|
||||
return False
|
||||
|
||||
@@ -757,6 +780,66 @@ class ServerArgsAutoTuner:
|
||||
return True
|
||||
return self._dit_overflows_device_in_auto(deployment_config)
|
||||
|
||||
def _keeps_all_components_resident(self) -> bool:
|
||||
if self._all_resident is None:
|
||||
self._all_resident = self._fits_all_components_resident()
|
||||
return self._all_resident
|
||||
|
||||
def _fits_all_components_resident(self) -> bool:
|
||||
"""Memory-rich auto tier: the least-free GPU holds every weight twice over.
|
||||
|
||||
Sized from the local checkpoint, so overrides, quantization and weights not
|
||||
yet downloaded keep the per-model policy; one request's activations are
|
||||
assumed, so batching and shared host/device pools are excluded too.
|
||||
"""
|
||||
args = self.server_args
|
||||
if (
|
||||
args.performance_mode != "auto"
|
||||
or current_platform.is_cpu()
|
||||
or current_platform.is_mps()
|
||||
or current_platform.device_shares_host_memory()
|
||||
or args.batching_max_size > 1
|
||||
or args.quantization is not None
|
||||
or args.transformer_weights_path is not None
|
||||
or _uses_svdquant(args.nunchaku_config)
|
||||
or args.component_paths
|
||||
or args.component_weights_paths
|
||||
or args.component_quantizations
|
||||
or resolve_model_overlay(args.model_path) is not None
|
||||
):
|
||||
return False
|
||||
available_gb = self._get_min_available_device_memory_gb()
|
||||
if available_gb is None:
|
||||
return False
|
||||
sizes = checkpoint_component_sizes(
|
||||
args.model_path, subfolder=args.model_subfolder, revision=args.revision
|
||||
)
|
||||
if sizes is None:
|
||||
return False
|
||||
weight_bytes = 0
|
||||
for name, (num_params, stored_bytes) in sizes.items():
|
||||
# loaded in the component's configured precision, else as stored
|
||||
dtype = resolve_component_precision(args, name)
|
||||
nbytes = num_params * dtype.itemsize if dtype is not None else stored_bytes
|
||||
# tensor parallelism shards the DiT's linear layers across the ranks
|
||||
weight_bytes += (
|
||||
nbytes / (args.tp_size or 1) if is_dit_component_name(name) else nbytes
|
||||
)
|
||||
weight_gb = weight_bytes / BYTES_PER_GB
|
||||
fits = available_gb >= AUTO_ALL_RESIDENT_WEIGHT_MULTIPLE * weight_gb
|
||||
if fits:
|
||||
logger.info(
|
||||
"Auto memory policy for %s: %.1f GiB free on the least-free "
|
||||
"selected GPU holds the %.1f GiB of weights per GPU at least "
|
||||
"%dx, so every component without an explicit placement stays "
|
||||
"resident.",
|
||||
args.pipeline_config.__class__.__name__,
|
||||
available_gb,
|
||||
weight_gb,
|
||||
AUTO_ALL_RESIDENT_WEIGHT_MULTIPLE,
|
||||
)
|
||||
return fits
|
||||
|
||||
def _dit_overflows_device_in_auto(
|
||||
self, deployment_config: ModelDeploymentConfig
|
||||
) -> bool:
|
||||
|
||||
@@ -326,7 +326,8 @@ ONE_GPU_CASES: list[DiffusionTestCase] = [
|
||||
# request the stage regains that whole transfer, well past the
|
||||
# non_denoise_stage tolerance. load_peak_vram carries the placed set, so
|
||||
# it also pins that the placement happens at load, not on the first use.
|
||||
# The encoder is already layerwise under the default component set.
|
||||
# The encoder is pinned layerwise: on an 80 GB card auto would keep the
|
||||
# whole 1.3B checkpoint resident.
|
||||
# No consistency check: the lifetime moves weights, not math, so the
|
||||
# output is the base case's and that case already guards it.
|
||||
DiffusionTestCase(
|
||||
@@ -334,6 +335,7 @@ ONE_GPU_CASES: list[DiffusionTestCase] = [
|
||||
DiffusionServerArgs(
|
||||
model_path=DEFAULT_WAN_2_1_T2V_1_3B_MODEL_NAME_FOR_TEST,
|
||||
extras=[
|
||||
"--component-residency text_encoder=layerwise-offload "
|
||||
"--layerwise-resident-layers text_encoder=0.8 "
|
||||
"--layerwise-residency-lifetime text_encoder=permanent",
|
||||
],
|
||||
|
||||
@@ -272,10 +272,10 @@
|
||||
"expected_avg_denoise_ms": 478.63,
|
||||
"expected_median_denoise_ms": 443.42,
|
||||
"estimated_full_test_time_s": 181.8,
|
||||
"load_peak_vram_mb": 4920.72,
|
||||
"runtime_peak_vram_mb": 9895.98,
|
||||
"load_peak_allocated_mb": 4858.5,
|
||||
"runtime_peak_allocated_mb": 6460.38
|
||||
"load_peak_vram_mb": 25490.0,
|
||||
"runtime_peak_vram_mb": 35568.0,
|
||||
"load_peak_allocated_mb": 24622.71,
|
||||
"runtime_peak_allocated_mb": 28881.55
|
||||
},
|
||||
"ideogram4_nvfp4_t2i": {
|
||||
"expected_load_ms": 33439.28,
|
||||
|
||||
@@ -465,10 +465,10 @@
|
||||
"expected_e2e_ms": 6693.01,
|
||||
"expected_avg_denoise_ms": 128.85,
|
||||
"expected_median_denoise_ms": 130.22,
|
||||
"load_peak_vram_mb": 23594.0,
|
||||
"runtime_peak_vram_mb": 28540.0,
|
||||
"load_peak_allocated_mb": 23572.0,
|
||||
"runtime_peak_allocated_mb": 25667.0,
|
||||
"load_peak_vram_mb": 32296.0,
|
||||
"runtime_peak_vram_mb": 36766.0,
|
||||
"load_peak_allocated_mb": 32191.79,
|
||||
"runtime_peak_allocated_mb": 34669.69,
|
||||
"estimated_full_test_time_s": 80
|
||||
},
|
||||
"flux_2_image_t2i": {
|
||||
@@ -850,10 +850,10 @@
|
||||
"expected_e2e_ms": 4244.27,
|
||||
"expected_avg_denoise_ms": 79.98,
|
||||
"expected_median_denoise_ms": 81.01,
|
||||
"load_peak_vram_mb": 23284.0,
|
||||
"runtime_peak_vram_mb": 28148.0,
|
||||
"load_peak_allocated_mb": 23262.0,
|
||||
"runtime_peak_allocated_mb": 25541.0,
|
||||
"load_peak_vram_mb": 27722.0,
|
||||
"runtime_peak_vram_mb": 32332.0,
|
||||
"load_peak_allocated_mb": 27651.29,
|
||||
"runtime_peak_allocated_mb": 30128.94,
|
||||
"estimated_full_test_time_s": 50.0
|
||||
},
|
||||
"zimage_image_t2i": {
|
||||
@@ -880,10 +880,10 @@
|
||||
"expected_e2e_ms": 906.14,
|
||||
"expected_avg_denoise_ms": 82.61,
|
||||
"expected_median_denoise_ms": 90.0,
|
||||
"load_peak_vram_mb": 13120.0,
|
||||
"runtime_peak_vram_mb": 17776.0,
|
||||
"load_peak_allocated_mb": 12852.0,
|
||||
"runtime_peak_allocated_mb": 15133.0,
|
||||
"load_peak_vram_mb": 20546.0,
|
||||
"runtime_peak_vram_mb": 24492.0,
|
||||
"load_peak_allocated_mb": 20217.47,
|
||||
"runtime_peak_allocated_mb": 22117.63,
|
||||
"estimated_full_test_time_s": 50
|
||||
},
|
||||
"zimage_image_t2i_fp8": {
|
||||
@@ -940,10 +940,10 @@
|
||||
"expected_e2e_ms": 910.85,
|
||||
"expected_avg_denoise_ms": 83.08,
|
||||
"expected_median_denoise_ms": 89.93,
|
||||
"load_peak_vram_mb": 14196.0,
|
||||
"runtime_peak_vram_mb": 17612.0,
|
||||
"load_peak_allocated_mb": 13059.0,
|
||||
"runtime_peak_allocated_mb": 15307.0,
|
||||
"load_peak_vram_mb": 21210.0,
|
||||
"runtime_peak_vram_mb": 24626.0,
|
||||
"load_peak_allocated_mb": 20453.97,
|
||||
"runtime_peak_allocated_mb": 22291.56,
|
||||
"estimated_full_test_time_s": 90
|
||||
},
|
||||
"zimage_image_t2i_2_gpus": {
|
||||
@@ -970,10 +970,10 @@
|
||||
"expected_e2e_ms": 752.84,
|
||||
"expected_avg_denoise_ms": 51.13,
|
||||
"expected_median_denoise_ms": 54.3,
|
||||
"load_peak_vram_mb": 13120.0,
|
||||
"runtime_peak_vram_mb": 17602.0,
|
||||
"load_peak_allocated_mb": 12852.0,
|
||||
"runtime_peak_allocated_mb": 15131.0,
|
||||
"load_peak_vram_mb": 20546.0,
|
||||
"runtime_peak_vram_mb": 24340.0,
|
||||
"load_peak_allocated_mb": 20217.47,
|
||||
"runtime_peak_allocated_mb": 22115.25,
|
||||
"estimated_full_test_time_s": 50.0
|
||||
},
|
||||
"qwen_image_edit_ti2i": {
|
||||
@@ -1246,10 +1246,10 @@
|
||||
"expected_e2e_ms": 4739.06,
|
||||
"expected_avg_denoise_ms": 76.99,
|
||||
"expected_median_denoise_ms": 98.28,
|
||||
"load_peak_vram_mb": 7814.0,
|
||||
"runtime_peak_vram_mb": 18078.0,
|
||||
"load_peak_allocated_mb": 3641.0,
|
||||
"runtime_peak_allocated_mb": 9730.0,
|
||||
"load_peak_vram_mb": 26126.0,
|
||||
"runtime_peak_vram_mb": 33340.0,
|
||||
"load_peak_allocated_mb": 25740.95,
|
||||
"runtime_peak_allocated_mb": 31408.67,
|
||||
"estimated_full_test_time_s": 60
|
||||
},
|
||||
"wan2_1_t2v_1.3b_encoder_permanent_residents": {
|
||||
@@ -1389,10 +1389,10 @@
|
||||
"expected_e2e_ms": 8106.87,
|
||||
"expected_avg_denoise_ms": 143.82,
|
||||
"expected_median_denoise_ms": 144.57,
|
||||
"load_peak_vram_mb": 7814.0,
|
||||
"runtime_peak_vram_mb": 18078.0,
|
||||
"load_peak_allocated_mb": 3641.0,
|
||||
"runtime_peak_allocated_mb": 9673.0,
|
||||
"load_peak_vram_mb": 26126.0,
|
||||
"runtime_peak_vram_mb": 33340.0,
|
||||
"load_peak_allocated_mb": 25740.95,
|
||||
"runtime_peak_allocated_mb": 31354.62,
|
||||
"estimated_full_test_time_s": 150
|
||||
},
|
||||
"wan2_1_t2v_1.3b_cfg_parallel": {
|
||||
@@ -1460,10 +1460,10 @@
|
||||
"expected_e2e_ms": 4435.63,
|
||||
"expected_avg_denoise_ms": 73.51,
|
||||
"expected_median_denoise_ms": 74.05,
|
||||
"load_peak_vram_mb": 5422.0,
|
||||
"runtime_peak_vram_mb": 11972.0,
|
||||
"load_peak_allocated_mb": 5274.0,
|
||||
"runtime_peak_allocated_mb": 9251.0,
|
||||
"load_peak_vram_mb": 15292.0,
|
||||
"runtime_peak_vram_mb": 19728.0,
|
||||
"load_peak_allocated_mb": 14906.86,
|
||||
"runtime_peak_allocated_mb": 18094.7,
|
||||
"estimated_full_test_time_s": 60.0
|
||||
},
|
||||
"turbo_wan2_1_t2v_1.3b": {
|
||||
@@ -1629,10 +1629,10 @@
|
||||
"expected_e2e_ms": 15599.56,
|
||||
"expected_avg_denoise_ms": 253.29,
|
||||
"expected_median_denoise_ms": 256.21,
|
||||
"load_peak_vram_mb": 17610.0,
|
||||
"runtime_peak_vram_mb": 39856.0,
|
||||
"load_peak_allocated_mb": 11629.0,
|
||||
"runtime_peak_allocated_mb": 26066.0,
|
||||
"load_peak_vram_mb": 36644.0,
|
||||
"runtime_peak_vram_mb": 53216.0,
|
||||
"load_peak_allocated_mb": 36480.46,
|
||||
"runtime_peak_allocated_mb": 49073.06,
|
||||
"estimated_full_test_time_s": 80
|
||||
},
|
||||
"qwen_image_edit_2509_ti2i": {
|
||||
@@ -1784,10 +1784,10 @@
|
||||
"expected_e2e_ms": 3073.51,
|
||||
"expected_avg_denoise_ms": 83.86,
|
||||
"expected_median_denoise_ms": 112.67,
|
||||
"load_peak_vram_mb": 17610.0,
|
||||
"runtime_peak_vram_mb": 39054.0,
|
||||
"load_peak_allocated_mb": 11629.0,
|
||||
"runtime_peak_allocated_mb": 26051.0,
|
||||
"load_peak_vram_mb": 36644.0,
|
||||
"runtime_peak_vram_mb": 53636.0,
|
||||
"load_peak_allocated_mb": 36480.46,
|
||||
"runtime_peak_allocated_mb": 49057.2,
|
||||
"estimated_full_test_time_s": 60
|
||||
},
|
||||
"fast_hunyuan_video": {
|
||||
@@ -1812,11 +1812,11 @@
|
||||
"expected_e2e_ms": 9123.68,
|
||||
"expected_avg_denoise_ms": 871.39,
|
||||
"expected_median_denoise_ms": 1010.9,
|
||||
"load_peak_vram_mb": 27320.0,
|
||||
"runtime_peak_vram_mb": 35926.0,
|
||||
"load_peak_vram_mb": 40050.0,
|
||||
"runtime_peak_vram_mb": 47480.0,
|
||||
"warmup_peak_vram_mb": 56748.0,
|
||||
"load_peak_allocated_mb": 26241.0,
|
||||
"runtime_peak_allocated_mb": 31922.0,
|
||||
"load_peak_allocated_mb": 39773.7,
|
||||
"runtime_peak_allocated_mb": 44331.45,
|
||||
"estimated_full_test_time_s": 80
|
||||
},
|
||||
"wan2_2_i2v_a14b_2gpu": {
|
||||
@@ -2287,10 +2287,10 @@
|
||||
"expected_e2e_ms": 8138.0,
|
||||
"expected_avg_denoise_ms": 146.48,
|
||||
"expected_median_denoise_ms": 147.68,
|
||||
"load_peak_vram_mb": 7814.0,
|
||||
"runtime_peak_vram_mb": 18350.0,
|
||||
"load_peak_allocated_mb": 3641.0,
|
||||
"runtime_peak_allocated_mb": 9999.0,
|
||||
"load_peak_vram_mb": 26126.0,
|
||||
"runtime_peak_vram_mb": 33442.0,
|
||||
"load_peak_allocated_mb": 25740.95,
|
||||
"runtime_peak_allocated_mb": 31666.73,
|
||||
"estimated_full_test_time_s": 90
|
||||
},
|
||||
"wan2_1_t2v_1_3b_dynamic_lora_default_quality_1gpu": {
|
||||
@@ -2557,10 +2557,10 @@
|
||||
"expected_e2e_ms": 1042.43,
|
||||
"expected_avg_denoise_ms": 82.26,
|
||||
"expected_median_denoise_ms": 81.75,
|
||||
"load_peak_vram_mb": 7176.0,
|
||||
"runtime_peak_vram_mb": 12634.0,
|
||||
"load_peak_allocated_mb": 7033.0,
|
||||
"runtime_peak_allocated_mb": 9311.0,
|
||||
"load_peak_vram_mb": 14558.0,
|
||||
"runtime_peak_vram_mb": 19230.0,
|
||||
"load_peak_allocated_mb": 14399.24,
|
||||
"runtime_peak_allocated_mb": 16295.65,
|
||||
"estimated_full_test_time_s": 40.0
|
||||
},
|
||||
"hunyuan3d_shape_gen": {
|
||||
@@ -2700,10 +2700,10 @@
|
||||
"expected_e2e_ms": 8273.83,
|
||||
"expected_avg_denoise_ms": 146.61,
|
||||
"expected_median_denoise_ms": 147.3,
|
||||
"load_peak_vram_mb": 7814.0,
|
||||
"runtime_peak_vram_mb": 18080.0,
|
||||
"load_peak_allocated_mb": 3641.0,
|
||||
"runtime_peak_allocated_mb": 9673.0,
|
||||
"load_peak_vram_mb": 26126.0,
|
||||
"runtime_peak_vram_mb": 34026.0,
|
||||
"load_peak_allocated_mb": 25740.95,
|
||||
"runtime_peak_allocated_mb": 31340.91,
|
||||
"estimated_full_test_time_s": 70
|
||||
},
|
||||
"flux_2_image_t2i_upscaling_4x": {
|
||||
@@ -2843,10 +2843,10 @@
|
||||
"expected_e2e_ms": 8333.66,
|
||||
"expected_avg_denoise_ms": 144.68,
|
||||
"expected_median_denoise_ms": 144.74,
|
||||
"load_peak_vram_mb": 7814.0,
|
||||
"runtime_peak_vram_mb": 24380.0,
|
||||
"load_peak_allocated_mb": 3641.0,
|
||||
"runtime_peak_allocated_mb": 10508.0,
|
||||
"load_peak_vram_mb": 26126.0,
|
||||
"runtime_peak_vram_mb": 39714.0,
|
||||
"load_peak_allocated_mb": 25740.95,
|
||||
"runtime_peak_allocated_mb": 31340.91,
|
||||
"estimated_full_test_time_s": 100
|
||||
},
|
||||
"wan2_1_t2v_1.3b_frame_interp_2x_upscaling_4x": {
|
||||
@@ -2914,10 +2914,10 @@
|
||||
"expected_e2e_ms": 8099.03,
|
||||
"expected_avg_denoise_ms": 143.71,
|
||||
"expected_median_denoise_ms": 144.67,
|
||||
"load_peak_vram_mb": 7814.0,
|
||||
"runtime_peak_vram_mb": 24388.0,
|
||||
"load_peak_allocated_mb": 3641.0,
|
||||
"runtime_peak_allocated_mb": 10531.0,
|
||||
"load_peak_vram_mb": 26126.0,
|
||||
"runtime_peak_vram_mb": 40406.0,
|
||||
"load_peak_allocated_mb": 25740.95,
|
||||
"runtime_peak_allocated_mb": 31340.91,
|
||||
"estimated_full_test_time_s": 100
|
||||
},
|
||||
"ltx_2.3_one_stage_ti2v": {
|
||||
|
||||
@@ -127,6 +127,11 @@ def mock_cuda_platform(
|
||||
"sglang.multimodal_gen.runtime.platforms.current_platform.enable_dit_layerwise_offload_by_default",
|
||||
return_value=True,
|
||||
),
|
||||
# checkpoint sizing reads the local HF cache; tests that need it pass a size
|
||||
patch(
|
||||
"sglang.multimodal_gen.runtime.server_args.auto_tune.checkpoint_component_sizes",
|
||||
return_value=None,
|
||||
),
|
||||
):
|
||||
yield
|
||||
|
||||
|
||||
@@ -1232,6 +1232,8 @@ class TestOffloadDefaults(_CudaPlatformTestCase):
|
||||
memory_gb=80,
|
||||
available_memory_gb=None,
|
||||
dit_parameter_count=None,
|
||||
checkpoint_sizes=None,
|
||||
is_cuda=True,
|
||||
kwargs=None,
|
||||
):
|
||||
def get_available_gpu_memory(device_id=0, **_kwargs):
|
||||
@@ -1251,6 +1253,10 @@ class TestOffloadDefaults(_CudaPlatformTestCase):
|
||||
else Mock(return_value=dit_parameter_count)
|
||||
),
|
||||
),
|
||||
patch(
|
||||
"sglang.multimodal_gen.runtime.server_args.auto_tune.checkpoint_component_sizes",
|
||||
return_value=checkpoint_sizes,
|
||||
),
|
||||
patch(
|
||||
"sglang.multimodal_gen.runtime.platforms.current_platform.is_cpu",
|
||||
return_value=False,
|
||||
@@ -1261,7 +1267,7 @@ class TestOffloadDefaults(_CudaPlatformTestCase):
|
||||
),
|
||||
patch(
|
||||
"sglang.multimodal_gen.runtime.platforms.current_platform.is_cuda",
|
||||
return_value=True,
|
||||
return_value=is_cuda,
|
||||
),
|
||||
patch(
|
||||
"sglang.multimodal_gen.runtime.platforms.current_platform.enable_dit_layerwise_offload_by_default",
|
||||
@@ -2107,6 +2113,101 @@ class TestOffloadDefaults(_CudaPlatformTestCase):
|
||||
["text_encoder", "image_encoder"],
|
||||
)
|
||||
|
||||
def _zimage_turbo(
|
||||
self, *, memory_gb=141, available_memory_gb=139, checkpoint_sizes=None, **kwargs
|
||||
):
|
||||
return self._from_dict_with_pipeline_config(
|
||||
ZImagePipelineConfig(),
|
||||
memory_gb=memory_gb,
|
||||
available_memory_gb=available_memory_gb,
|
||||
checkpoint_sizes=checkpoint_sizes,
|
||||
kwargs={
|
||||
"model_path": "Tongyi-MAI/Z-Image-Turbo",
|
||||
"performance_mode": "auto",
|
||||
**kwargs,
|
||||
},
|
||||
)
|
||||
|
||||
# (parameters, stored bytes) of Tongyi-MAI/Z-Image-Turbo: the DiT is stored in
|
||||
# fp32 (22.9 GiB) and served in bf16, 19.3 GiB per GPU in all
|
||||
ZIMAGE_TURBO_SIZES = {
|
||||
"transformer": (6_154_908_736, 24_619_634_944),
|
||||
"text_encoder": (4_022_468_096, 8_044_936_192),
|
||||
"vae": (83_819_683, 335_278_732),
|
||||
}
|
||||
|
||||
def _residency(self, args):
|
||||
return {
|
||||
name: args.residency_mode(name)
|
||||
for name in ("transformer", "text_encoder", "image_encoder", "vae")
|
||||
}
|
||||
|
||||
def test_auto_keeps_every_component_resident_when_weights_fit_twice(self):
|
||||
args = self._zimage_turbo(checkpoint_sizes=self.ZIMAGE_TURBO_SIZES)
|
||||
|
||||
self.assertIsNone(args.layerwise_offload_components)
|
||||
self.assertFalse(args.dit_cpu_offload)
|
||||
self.assertFalse(args.text_encoder_cpu_offload)
|
||||
self.assertEqual(set(self._residency(args).values()), {RESIDENT})
|
||||
|
||||
def test_auto_tier_sizes_weights_in_their_serving_precision(self):
|
||||
# fits twice in 45 GiB at bf16; the fp32-stored bytes would not
|
||||
args = self._zimage_turbo(
|
||||
memory_gb=48,
|
||||
available_memory_gb=45,
|
||||
checkpoint_sizes=self.ZIMAGE_TURBO_SIZES,
|
||||
)
|
||||
self.assertEqual(set(self._residency(args).values()), {RESIDENT})
|
||||
|
||||
def test_auto_tier_leaves_consumer_gpus_on_their_policy(self):
|
||||
for memory_gb, available_gb in ((32, 31), (24, 23)):
|
||||
with self.subTest(memory_gb=memory_gb):
|
||||
sized = self._zimage_turbo(
|
||||
memory_gb=memory_gb,
|
||||
available_memory_gb=available_gb,
|
||||
checkpoint_sizes=self.ZIMAGE_TURBO_SIZES,
|
||||
)
|
||||
unsized = self._zimage_turbo(
|
||||
memory_gb=memory_gb, available_memory_gb=available_gb
|
||||
)
|
||||
self.assertEqual(self._residency(sized), self._residency(unsized))
|
||||
self.assertNotEqual(sized.residency_mode("text_encoder"), RESIDENT)
|
||||
|
||||
def test_auto_tier_skips_batching_shared_memory_and_unsized_checkpoints(self):
|
||||
unsized = self._zimage_turbo()
|
||||
self.assertEqual(unsized.residency_mode("text_encoder"), LAYERWISE_OFFLOAD)
|
||||
batching = self._zimage_turbo(
|
||||
checkpoint_sizes=self.ZIMAGE_TURBO_SIZES, batching_max_size=2
|
||||
)
|
||||
self.assertEqual(batching.residency_mode("text_encoder"), LAYERWISE_OFFLOAD)
|
||||
with patch.object(
|
||||
current_platform, "device_shares_host_memory", return_value=True
|
||||
):
|
||||
shared = self._zimage_turbo(checkpoint_sizes=self.ZIMAGE_TURBO_SIZES)
|
||||
self.assertEqual(shared.residency_mode("text_encoder"), LAYERWISE_OFFLOAD)
|
||||
|
||||
def test_auto_tier_replaces_whole_component_offload_off_cuda(self):
|
||||
# off CUDA auto has no layerwise defaults and offloads the encoder whole
|
||||
unsized = self._from_dict_with_pipeline_config(
|
||||
ZImagePipelineConfig(),
|
||||
memory_gb=192,
|
||||
available_memory_gb=190,
|
||||
is_cuda=False,
|
||||
kwargs={"model_path": "Tongyi-MAI/Z-Image-Turbo"},
|
||||
)
|
||||
sized = self._from_dict_with_pipeline_config(
|
||||
ZImagePipelineConfig(),
|
||||
memory_gb=192,
|
||||
available_memory_gb=190,
|
||||
checkpoint_sizes=self.ZIMAGE_TURBO_SIZES,
|
||||
is_cuda=False,
|
||||
kwargs={"model_path": "Tongyi-MAI/Z-Image-Turbo"},
|
||||
)
|
||||
|
||||
self.assertTrue(unsized.text_encoder_cpu_offload)
|
||||
self.assertFalse(sized.text_encoder_cpu_offload)
|
||||
self.assertEqual(set(self._residency(sized).values()), {RESIDENT})
|
||||
|
||||
def test_auto_lingbot_keeps_dit_resident_on_h100(self):
|
||||
args = self._from_dict_with_pipeline_config(
|
||||
LingBotWorldCausalDMDConfig(),
|
||||
@@ -2215,6 +2316,45 @@ class TestOffloadDefaults(_CudaPlatformTestCase):
|
||||
["dit", "text_encoder", "image_encoder", "vae"],
|
||||
)
|
||||
|
||||
def test_auto_wan2_2_a14b_keeps_both_experts_resident_above_120gb(self):
|
||||
for pipeline_config, model_path in (
|
||||
(Wan2_2_T2V_A14B_Config(), "Wan-AI/Wan2.2-T2V-A14B-Diffusers"),
|
||||
(Wan2_2_I2V_A14B_Config(), "Wan-AI/Wan2.2-I2V-A14B-Diffusers"),
|
||||
):
|
||||
with self.subTest(pipeline_config=pipeline_config.__class__.__name__):
|
||||
args = self._from_dict_with_pipeline_config(
|
||||
pipeline_config,
|
||||
memory_gb=141,
|
||||
available_memory_gb=139,
|
||||
kwargs={"model_path": model_path, "performance_mode": "auto"},
|
||||
)
|
||||
|
||||
self.assertFalse(args.dit_cpu_offload)
|
||||
self.assertEqual(args.residency_mode("transformer"), RESIDENT)
|
||||
self.assertEqual(args.residency_mode("transformer_2"), RESIDENT)
|
||||
self.assertEqual(args.residency_mode("vae"), RESIDENT)
|
||||
self.assertEqual(
|
||||
args.layerwise_offload_components,
|
||||
["text_encoder", "image_encoder"],
|
||||
)
|
||||
|
||||
# two 14B experts plus T5 fit twice over: the encoder joins them
|
||||
rich = self._from_dict_with_pipeline_config(
|
||||
pipeline_config,
|
||||
memory_gb=288,
|
||||
available_memory_gb=280,
|
||||
checkpoint_sizes={
|
||||
"transformer": (14_288_901_184, 57_155_604_736),
|
||||
"transformer_2": (14_288_901_184, 57_155_604_736),
|
||||
"text_encoder": (5_680_910_336, 22_723_641_344),
|
||||
"vae": (126_892_531, 507_570_124),
|
||||
},
|
||||
kwargs={"model_path": model_path, "performance_mode": "auto"},
|
||||
)
|
||||
self.assertIsNone(rich.layerwise_offload_components)
|
||||
self.assertEqual(rich.residency_mode("text_encoder"), RESIDENT)
|
||||
self.assertEqual(rich.residency_mode("transformer_2"), RESIDENT)
|
||||
|
||||
def test_auto_wan2_1_14b_keeps_dit_resident_on_h100(self):
|
||||
for pipeline_config, model_path in (
|
||||
(WanT2V720PConfig(), "Wan-AI/Wan2.1-T2V-14B-Diffusers"),
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
"""Unit tests for multimodal weight-loading utilities."""
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
@@ -15,6 +16,7 @@ from safetensors.torch import save_file as safetensors_save_file
|
||||
from sglang.multimodal_gen.runtime.loader.utils import (
|
||||
_list_safetensors_files,
|
||||
checkpoint_bytes,
|
||||
checkpoint_component_sizes,
|
||||
dit_parameter_count,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.loader.weight_utils import (
|
||||
@@ -266,6 +268,79 @@ class TestDitParameterCount(CustomTestCase):
|
||||
self.assertEqual(hub_download.called, use_modelscope == "0")
|
||||
|
||||
|
||||
def _write_pipeline(root: Path) -> None:
|
||||
"""A Diffusers layout: a 58-byte DiT, a 16-byte encoder and weightless parts."""
|
||||
_write_sharded_transformer(root)
|
||||
(root / "text_encoder").mkdir()
|
||||
safetensors_save_file(
|
||||
{"e": torch.zeros(4)}, root / "text_encoder" / "model.safetensors"
|
||||
)
|
||||
for weightless in ("tokenizer", "scheduler"):
|
||||
(root / weightless).mkdir()
|
||||
(root / weightless / "config.json").write_text("{}")
|
||||
(root / "model_index.json").write_text(
|
||||
'{"_class_name": "P", "transformer": ["diffusers", "T"], '
|
||||
'"text_encoder": ["transformers", "E"], "tokenizer": ["transformers", "K"], '
|
||||
'"scheduler": ["diffusers", "S"], "safety_checker": [null, null], '
|
||||
'"boundary_ratio": 0.9}'
|
||||
)
|
||||
|
||||
|
||||
class TestCheckpointComponentSizes(CustomTestCase):
|
||||
SIZES = {"transformer": (17, 58), "text_encoder": (4, 16)}
|
||||
|
||||
def test_sizes_every_weighted_component(self):
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
_write_pipeline(Path(tmpdir) / "sub")
|
||||
|
||||
self.assertEqual(
|
||||
checkpoint_component_sizes(tmpdir, subfolder="sub", revision=None),
|
||||
self.SIZES,
|
||||
)
|
||||
|
||||
def test_cached_hub_snapshot_is_sized_without_the_network(self):
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
_write_pipeline(Path(tmpdir))
|
||||
with patch(_SNAPSHOT_DOWNLOAD, return_value=tmpdir) as snapshot_download:
|
||||
self.assertEqual(
|
||||
checkpoint_component_sizes(
|
||||
"org/model", subfolder=None, revision="abc"
|
||||
),
|
||||
self.SIZES,
|
||||
)
|
||||
self.assertTrue(snapshot_download.call_args.kwargs["local_files_only"])
|
||||
|
||||
def test_partial_download_other_formats_or_quantized_weights_are_unsized(self):
|
||||
cases = {
|
||||
# only model_index.json fetched so far, as before the worker downloads
|
||||
"index only": lambda root: shutil.rmtree(root / "text_encoder"),
|
||||
"missing shard": lambda root: (
|
||||
root / "transformer" / "shard-2.safetensors"
|
||||
).unlink(),
|
||||
"bin weights": lambda root: (
|
||||
(root / "text_encoder" / "model.safetensors").unlink(),
|
||||
(root / "text_encoder" / "pytorch_model.bin").write_bytes(b"0"),
|
||||
),
|
||||
"quantized": lambda root: safetensors_save_file(
|
||||
{"q": torch.zeros(4, dtype=torch.float8_e4m3fn)},
|
||||
root / "text_encoder" / "model.safetensors",
|
||||
),
|
||||
}
|
||||
for name, damage in cases.items():
|
||||
with self.subTest(name), tempfile.TemporaryDirectory() as tmpdir:
|
||||
_write_pipeline(Path(tmpdir))
|
||||
damage(Path(tmpdir))
|
||||
self.assertIsNone(
|
||||
checkpoint_component_sizes(tmpdir, subfolder=None, revision=None)
|
||||
)
|
||||
|
||||
def test_uncached_repo_is_unsized(self):
|
||||
with patch(_SNAPSHOT_DOWNLOAD, side_effect=LocalEntryNotFoundError("miss")):
|
||||
self.assertIsNone(
|
||||
checkpoint_component_sizes("org/model", subfolder=None, revision=None)
|
||||
)
|
||||
|
||||
|
||||
class TestDisableRunaiStreamerRankDiscoveryCollective(unittest.TestCase):
|
||||
def test_never_touches_torch_distributed_even_when_initialized(self):
|
||||
from runai_model_streamer.distributed_streamer.distributed_streamer import (
|
||||
|
||||
Reference in New Issue
Block a user