diff --git a/docs/docs/sglang-diffusion/deployment_cookbook.mdx b/docs/docs/sglang-diffusion/deployment_cookbook.mdx index d7f911c29b..2a7c3334e0 100644 --- a/docs/docs/sglang-diffusion/deployment_cookbook.mdx +++ b/docs/docs/sglang-diffusion/deployment_cookbook.mdx @@ -115,7 +115,7 @@ See [OpenAI API: Served model name](/docs/sglang-diffusion/api/openai_api#served `auto` - Default. Keeps legacy safe offload defaults and uses FSDP/CFG only on validated multi-GPU deployments where FSDP can replace DiT offload. + Default. Keeps every component resident when the least-free GPU has at least twice the model's per-GPU weights free (sized from the cached checkpoint; not with batching or on shared-memory devices). Otherwise keeps legacy safe offload defaults and uses FSDP/CFG only on validated multi-GPU deployments where FSDP can replace DiT offload. `speed` diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/wan.py b/python/sglang/multimodal_gen/configs/pipeline_configs/wan.py index 711a1bd180..5421cdc5c9 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/wan.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/wan.py @@ -288,9 +288,12 @@ class Wan2_2_T2V_A14B_Config(WanT2V480PConfig): self.dit_config.torch_compile_mode = "default" def get_model_deployment_config(self) -> ModelDeploymentConfig: + # both 14B experts plus T5 take ~66 GB per GPU; 120 keeps 80 GB cards streaming return ModelDeploymentConfig( dit_layerwise_offload_modes=("auto", "memory"), auto_dit_offload_prefetch_size=2, + keep_resident_min_available_gb=120, + keep_resident_components=("dit", "vae"), ) @@ -305,9 +308,12 @@ class Wan2_2_I2V_A14B_Config(WanI2V720PConfig): self.dit_config.boundary_ratio = self.boundary_ratio def get_model_deployment_config(self) -> ModelDeploymentConfig: + # both 14B experts plus T5 take ~66 GB per GPU; 120 keeps 80 GB cards streaming return ModelDeploymentConfig( dit_layerwise_offload_modes=("auto", "memory"), auto_dit_offload_prefetch_size=2, + keep_resident_min_available_gb=120, + keep_resident_components=("dit", "vae"), ) diff --git a/python/sglang/multimodal_gen/runtime/loader/utils.py b/python/sglang/multimodal_gen/runtime/loader/utils.py index f38f006126..0389a191ab 100644 --- a/python/sglang/multimodal_gen/runtime/loader/utils.py +++ b/python/sglang/multimodal_gen/runtime/loader/utils.py @@ -419,7 +419,9 @@ def checkpoint_bytes(model_path: str) -> int: _DIT_SUBFOLDER = "transformer" _DIT_SINGLE_FILE = "diffusion_pytorch_model.safetensors" -_FLOAT_DTYPES = frozenset({"F64", "F32", "F16", "BF16"}) +_FLOAT_DTYPE_BYTES = {"F64": 8, "F32": 4, "F16": 2, "BF16": 2} +_FLOAT_DTYPES = frozenset(_FLOAT_DTYPE_BYTES) +_UNSIZED_WEIGHT_SUFFIXES = (".bin", ".pt", ".pth", ".ckpt", ".gguf") def _index_shard_names(index_path: str) -> list[str]: @@ -507,6 +509,59 @@ def dit_parameter_count( return sum(counts.values()) +def checkpoint_component_sizes( + model_path: str, *, subfolder: str | None, revision: str | None +) -> dict[str, tuple[int, int]] | None: + """(parameter count, stored bytes) of each model_index.json component of a + locally cached Diffusers checkpoint, from safetensors headers. + + Weightless components (tokenizers, schedulers) are left out. None when any + component cannot be sized from local float safetensors (not cached, missing + shards, other formats, quantized weights). + """ + try: + model_path = os.path.expanduser(model_path) + if os.path.isdir(model_path): + root = model_path + elif os.path.exists(model_path) or envs.SGLANG_USE_MODELSCOPE.get(): + return None + else: + root = snapshot_download( + repo_id=model_path, + revision=revision, + allow_patterns=["*model_index.json"], + local_files_only=True, + ) + if subfolder: + root = os.path.join(root, subfolder) + with open(os.path.join(root, "model_index.json")) as f: + components = [ + name + for name, spec in json.load(f).items() + if isinstance(spec, list) and spec[0] is not None + ] + sizes = {} + for name in components: + folder = os.path.join(root, name) + # a component folder not yet downloaded raises here + files = os.listdir(folder) + if not any(file.endswith(".safetensors") for file in files): + if any(file.endswith(_UNSIZED_WEIGHT_SUFFIXES) for file in files): + return None + continue + counts = _local_dit_parameter_counts(folder) + if not counts or not counts.keys() <= _FLOAT_DTYPES: + return None + sizes[name] = ( + sum(counts.values()), + sum(n * _FLOAT_DTYPE_BYTES[dtype] for dtype, n in counts.items()), + ) + return sizes + except Exception as exc: + logger.debug("Could not size the checkpoint of %s: %s", model_path, exc) + return None + + def keep_checkpoint_mapped(*, weight_bytes: int, component: str) -> bool: """Whether a component's weights should stay on their file mapping. diff --git a/python/sglang/multimodal_gen/runtime/server_args/auto_tune.py b/python/sglang/multimodal_gen/runtime/server_args/auto_tune.py index cf012ceee5..f63e6a9860 100644 --- a/python/sglang/multimodal_gen/runtime/server_args/auto_tune.py +++ b/python/sglang/multimodal_gen/runtime/server_args/auto_tune.py @@ -18,6 +18,7 @@ from sglang.multimodal_gen.runtime.entrypoints.openai.realtime.registry import ( ) from sglang.multimodal_gen.runtime.loader.utils import ( BYTES_PER_GB, + checkpoint_component_sizes, dit_parameter_count, ) from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload_components import ( @@ -26,12 +27,16 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload_co LAYERWISE_OFFLOAD_IMAGE_ENCODER_GROUP, LAYERWISE_OFFLOAD_TEXT_ENCODER_GROUP, LAYERWISE_OFFLOAD_VAE_GROUP, + is_dit_component_name, normalize_layerwise_offload_components, ) from sglang.multimodal_gen.runtime.platforms import current_platform from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger from sglang.multimodal_gen.runtime.utils.model_overlay import resolve_model_overlay -from sglang.multimodal_gen.runtime.utils.precision import resolve_precision +from sglang.multimodal_gen.runtime.utils.precision import ( + resolve_component_precision, + resolve_precision, +) if TYPE_CHECKING: from sglang.multimodal_gen.runtime.server_args.server_args import ServerArgs @@ -40,6 +45,8 @@ logger = init_logger(__name__) # Picked, not measured; what stays free holds activations and the other components. AUTO_DIT_MAX_DEVICE_FRACTION = 0.8 +# auto keeps every component resident once the least-free GPU holds all weights this many times +AUTO_ALL_RESIDENT_WEIGHT_MULTIPLE = 2 def _uses_svdquant(nunchaku_config) -> bool: @@ -110,6 +117,12 @@ DEFAULT_LAYERWISE_COMPONENT_ARG_NAMES = ( # larger so it only stays resident on very-high-memory gpus IMAGE_GEN_KEEP_RESIDENT_MIN_AVAILABLE_GB = 45.0 DEFAULT_KEEP_RESIDENT_MIN_AVAILABLE_GB = 120.0 +ALL_OFFLOAD_COMPONENTS = ( + LAYERWISE_OFFLOAD_DIT_GROUP, + LAYERWISE_OFFLOAD_TEXT_ENCODER_GROUP, + LAYERWISE_OFFLOAD_IMAGE_ENCODER_GROUP, + LAYERWISE_OFFLOAD_VAE_GROUP, +) # torch's CPU allocator (mimalloc since 2.13) keeps freed pages in its arenas @@ -141,6 +154,8 @@ class ServerArgsAutoTuner: self.server_args = server_args self._explicit_dit_residency = self._has_explicit_dit_residency() self._dit_fit_checked = False + # memory-rich tier decision, resolved on first use + self._all_resident: bool | None = None def _deployment_config(self) -> ModelDeploymentConfig: return self.server_args.pipeline_config.get_model_deployment_config() @@ -242,13 +257,18 @@ class ServerArgsAutoTuner: disable_threshold_gb = self._resolve_keep_resident_min_available_gb( deployment_config ) - if ( + all_resident = self._keeps_all_components_resident() + if all_resident or ( min_available_gb is not None and disable_threshold_gb is not None and min_available_gb >= disable_threshold_gb ): changed = [] - components = set(deployment_config.keep_resident_components) + components = set( + ALL_OFFLOAD_COMPONENTS + if all_resident + else deployment_config.keep_resident_components + ) if args.pipeline_config.task_type.is_image_gen(): components.add(LAYERWISE_OFFLOAD_DIT_GROUP) if ( @@ -713,6 +733,8 @@ class ServerArgsAutoTuner: args = self.server_args if args.performance_mode != "auto" or current_platform.is_cpu(): return components + if self._keeps_all_components_resident(): + return [] deployment_config = self._deployment_config() threshold_gb = self._resolve_keep_resident_min_available_gb(deployment_config) @@ -749,6 +771,7 @@ class ServerArgsAutoTuner: or envs.SGLANG_CACHE_DIT_ENABLED or args.use_fsdp_inference or args.explicit_residency_mode("transformer") is not None + or self._keeps_all_components_resident() ): return False @@ -757,6 +780,66 @@ class ServerArgsAutoTuner: return True return self._dit_overflows_device_in_auto(deployment_config) + def _keeps_all_components_resident(self) -> bool: + if self._all_resident is None: + self._all_resident = self._fits_all_components_resident() + return self._all_resident + + def _fits_all_components_resident(self) -> bool: + """Memory-rich auto tier: the least-free GPU holds every weight twice over. + + Sized from the local checkpoint, so overrides, quantization and weights not + yet downloaded keep the per-model policy; one request's activations are + assumed, so batching and shared host/device pools are excluded too. + """ + args = self.server_args + if ( + args.performance_mode != "auto" + or current_platform.is_cpu() + or current_platform.is_mps() + or current_platform.device_shares_host_memory() + or args.batching_max_size > 1 + or args.quantization is not None + or args.transformer_weights_path is not None + or _uses_svdquant(args.nunchaku_config) + or args.component_paths + or args.component_weights_paths + or args.component_quantizations + or resolve_model_overlay(args.model_path) is not None + ): + return False + available_gb = self._get_min_available_device_memory_gb() + if available_gb is None: + return False + sizes = checkpoint_component_sizes( + args.model_path, subfolder=args.model_subfolder, revision=args.revision + ) + if sizes is None: + return False + weight_bytes = 0 + for name, (num_params, stored_bytes) in sizes.items(): + # loaded in the component's configured precision, else as stored + dtype = resolve_component_precision(args, name) + nbytes = num_params * dtype.itemsize if dtype is not None else stored_bytes + # tensor parallelism shards the DiT's linear layers across the ranks + weight_bytes += ( + nbytes / (args.tp_size or 1) if is_dit_component_name(name) else nbytes + ) + weight_gb = weight_bytes / BYTES_PER_GB + fits = available_gb >= AUTO_ALL_RESIDENT_WEIGHT_MULTIPLE * weight_gb + if fits: + logger.info( + "Auto memory policy for %s: %.1f GiB free on the least-free " + "selected GPU holds the %.1f GiB of weights per GPU at least " + "%dx, so every component without an explicit placement stays " + "resident.", + args.pipeline_config.__class__.__name__, + available_gb, + weight_gb, + AUTO_ALL_RESIDENT_WEIGHT_MULTIPLE, + ) + return fits + def _dit_overflows_device_in_auto( self, deployment_config: ModelDeploymentConfig ) -> bool: diff --git a/python/sglang/multimodal_gen/test/server/gpu_cases.py b/python/sglang/multimodal_gen/test/server/gpu_cases.py index 97de0ec4cd..41ec437f94 100644 --- a/python/sglang/multimodal_gen/test/server/gpu_cases.py +++ b/python/sglang/multimodal_gen/test/server/gpu_cases.py @@ -326,7 +326,8 @@ ONE_GPU_CASES: list[DiffusionTestCase] = [ # request the stage regains that whole transfer, well past the # non_denoise_stage tolerance. load_peak_vram carries the placed set, so # it also pins that the placement happens at load, not on the first use. - # The encoder is already layerwise under the default component set. + # The encoder is pinned layerwise: on an 80 GB card auto would keep the + # whole 1.3B checkpoint resident. # No consistency check: the lifetime moves weights, not math, so the # output is the base case's and that case already guards it. DiffusionTestCase( @@ -334,6 +335,7 @@ ONE_GPU_CASES: list[DiffusionTestCase] = [ DiffusionServerArgs( model_path=DEFAULT_WAN_2_1_T2V_1_3B_MODEL_NAME_FOR_TEST, extras=[ + "--component-residency text_encoder=layerwise-offload " "--layerwise-resident-layers text_encoder=0.8 " "--layerwise-residency-lifetime text_encoder=permanent", ], diff --git a/python/sglang/multimodal_gen/test/server/perf_baselines/b200.json b/python/sglang/multimodal_gen/test/server/perf_baselines/b200.json index 8b41fd60fb..46afb666f2 100644 --- a/python/sglang/multimodal_gen/test/server/perf_baselines/b200.json +++ b/python/sglang/multimodal_gen/test/server/perf_baselines/b200.json @@ -272,10 +272,10 @@ "expected_avg_denoise_ms": 478.63, "expected_median_denoise_ms": 443.42, "estimated_full_test_time_s": 181.8, - "load_peak_vram_mb": 4920.72, - "runtime_peak_vram_mb": 9895.98, - "load_peak_allocated_mb": 4858.5, - "runtime_peak_allocated_mb": 6460.38 + "load_peak_vram_mb": 25490.0, + "runtime_peak_vram_mb": 35568.0, + "load_peak_allocated_mb": 24622.71, + "runtime_peak_allocated_mb": 28881.55 }, "ideogram4_nvfp4_t2i": { "expected_load_ms": 33439.28, diff --git a/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json b/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json index 72208d860e..a1420df63f 100644 --- a/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json +++ b/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json @@ -465,10 +465,10 @@ "expected_e2e_ms": 6693.01, "expected_avg_denoise_ms": 128.85, "expected_median_denoise_ms": 130.22, - "load_peak_vram_mb": 23594.0, - "runtime_peak_vram_mb": 28540.0, - "load_peak_allocated_mb": 23572.0, - "runtime_peak_allocated_mb": 25667.0, + "load_peak_vram_mb": 32296.0, + "runtime_peak_vram_mb": 36766.0, + "load_peak_allocated_mb": 32191.79, + "runtime_peak_allocated_mb": 34669.69, "estimated_full_test_time_s": 80 }, "flux_2_image_t2i": { @@ -850,10 +850,10 @@ "expected_e2e_ms": 4244.27, "expected_avg_denoise_ms": 79.98, "expected_median_denoise_ms": 81.01, - "load_peak_vram_mb": 23284.0, - "runtime_peak_vram_mb": 28148.0, - "load_peak_allocated_mb": 23262.0, - "runtime_peak_allocated_mb": 25541.0, + "load_peak_vram_mb": 27722.0, + "runtime_peak_vram_mb": 32332.0, + "load_peak_allocated_mb": 27651.29, + "runtime_peak_allocated_mb": 30128.94, "estimated_full_test_time_s": 50.0 }, "zimage_image_t2i": { @@ -880,10 +880,10 @@ "expected_e2e_ms": 906.14, "expected_avg_denoise_ms": 82.61, "expected_median_denoise_ms": 90.0, - "load_peak_vram_mb": 13120.0, - "runtime_peak_vram_mb": 17776.0, - "load_peak_allocated_mb": 12852.0, - "runtime_peak_allocated_mb": 15133.0, + "load_peak_vram_mb": 20546.0, + "runtime_peak_vram_mb": 24492.0, + "load_peak_allocated_mb": 20217.47, + "runtime_peak_allocated_mb": 22117.63, "estimated_full_test_time_s": 50 }, "zimage_image_t2i_fp8": { @@ -940,10 +940,10 @@ "expected_e2e_ms": 910.85, "expected_avg_denoise_ms": 83.08, "expected_median_denoise_ms": 89.93, - "load_peak_vram_mb": 14196.0, - "runtime_peak_vram_mb": 17612.0, - "load_peak_allocated_mb": 13059.0, - "runtime_peak_allocated_mb": 15307.0, + "load_peak_vram_mb": 21210.0, + "runtime_peak_vram_mb": 24626.0, + "load_peak_allocated_mb": 20453.97, + "runtime_peak_allocated_mb": 22291.56, "estimated_full_test_time_s": 90 }, "zimage_image_t2i_2_gpus": { @@ -970,10 +970,10 @@ "expected_e2e_ms": 752.84, "expected_avg_denoise_ms": 51.13, "expected_median_denoise_ms": 54.3, - "load_peak_vram_mb": 13120.0, - "runtime_peak_vram_mb": 17602.0, - "load_peak_allocated_mb": 12852.0, - "runtime_peak_allocated_mb": 15131.0, + "load_peak_vram_mb": 20546.0, + "runtime_peak_vram_mb": 24340.0, + "load_peak_allocated_mb": 20217.47, + "runtime_peak_allocated_mb": 22115.25, "estimated_full_test_time_s": 50.0 }, "qwen_image_edit_ti2i": { @@ -1246,10 +1246,10 @@ "expected_e2e_ms": 4739.06, "expected_avg_denoise_ms": 76.99, "expected_median_denoise_ms": 98.28, - "load_peak_vram_mb": 7814.0, - "runtime_peak_vram_mb": 18078.0, - "load_peak_allocated_mb": 3641.0, - "runtime_peak_allocated_mb": 9730.0, + "load_peak_vram_mb": 26126.0, + "runtime_peak_vram_mb": 33340.0, + "load_peak_allocated_mb": 25740.95, + "runtime_peak_allocated_mb": 31408.67, "estimated_full_test_time_s": 60 }, "wan2_1_t2v_1.3b_encoder_permanent_residents": { @@ -1389,10 +1389,10 @@ "expected_e2e_ms": 8106.87, "expected_avg_denoise_ms": 143.82, "expected_median_denoise_ms": 144.57, - "load_peak_vram_mb": 7814.0, - "runtime_peak_vram_mb": 18078.0, - "load_peak_allocated_mb": 3641.0, - "runtime_peak_allocated_mb": 9673.0, + "load_peak_vram_mb": 26126.0, + "runtime_peak_vram_mb": 33340.0, + "load_peak_allocated_mb": 25740.95, + "runtime_peak_allocated_mb": 31354.62, "estimated_full_test_time_s": 150 }, "wan2_1_t2v_1.3b_cfg_parallel": { @@ -1460,10 +1460,10 @@ "expected_e2e_ms": 4435.63, "expected_avg_denoise_ms": 73.51, "expected_median_denoise_ms": 74.05, - "load_peak_vram_mb": 5422.0, - "runtime_peak_vram_mb": 11972.0, - "load_peak_allocated_mb": 5274.0, - "runtime_peak_allocated_mb": 9251.0, + "load_peak_vram_mb": 15292.0, + "runtime_peak_vram_mb": 19728.0, + "load_peak_allocated_mb": 14906.86, + "runtime_peak_allocated_mb": 18094.7, "estimated_full_test_time_s": 60.0 }, "turbo_wan2_1_t2v_1.3b": { @@ -1629,10 +1629,10 @@ "expected_e2e_ms": 15599.56, "expected_avg_denoise_ms": 253.29, "expected_median_denoise_ms": 256.21, - "load_peak_vram_mb": 17610.0, - "runtime_peak_vram_mb": 39856.0, - "load_peak_allocated_mb": 11629.0, - "runtime_peak_allocated_mb": 26066.0, + "load_peak_vram_mb": 36644.0, + "runtime_peak_vram_mb": 53216.0, + "load_peak_allocated_mb": 36480.46, + "runtime_peak_allocated_mb": 49073.06, "estimated_full_test_time_s": 80 }, "qwen_image_edit_2509_ti2i": { @@ -1784,10 +1784,10 @@ "expected_e2e_ms": 3073.51, "expected_avg_denoise_ms": 83.86, "expected_median_denoise_ms": 112.67, - "load_peak_vram_mb": 17610.0, - "runtime_peak_vram_mb": 39054.0, - "load_peak_allocated_mb": 11629.0, - "runtime_peak_allocated_mb": 26051.0, + "load_peak_vram_mb": 36644.0, + "runtime_peak_vram_mb": 53636.0, + "load_peak_allocated_mb": 36480.46, + "runtime_peak_allocated_mb": 49057.2, "estimated_full_test_time_s": 60 }, "fast_hunyuan_video": { @@ -1812,11 +1812,11 @@ "expected_e2e_ms": 9123.68, "expected_avg_denoise_ms": 871.39, "expected_median_denoise_ms": 1010.9, - "load_peak_vram_mb": 27320.0, - "runtime_peak_vram_mb": 35926.0, + "load_peak_vram_mb": 40050.0, + "runtime_peak_vram_mb": 47480.0, "warmup_peak_vram_mb": 56748.0, - "load_peak_allocated_mb": 26241.0, - "runtime_peak_allocated_mb": 31922.0, + "load_peak_allocated_mb": 39773.7, + "runtime_peak_allocated_mb": 44331.45, "estimated_full_test_time_s": 80 }, "wan2_2_i2v_a14b_2gpu": { @@ -2287,10 +2287,10 @@ "expected_e2e_ms": 8138.0, "expected_avg_denoise_ms": 146.48, "expected_median_denoise_ms": 147.68, - "load_peak_vram_mb": 7814.0, - "runtime_peak_vram_mb": 18350.0, - "load_peak_allocated_mb": 3641.0, - "runtime_peak_allocated_mb": 9999.0, + "load_peak_vram_mb": 26126.0, + "runtime_peak_vram_mb": 33442.0, + "load_peak_allocated_mb": 25740.95, + "runtime_peak_allocated_mb": 31666.73, "estimated_full_test_time_s": 90 }, "wan2_1_t2v_1_3b_dynamic_lora_default_quality_1gpu": { @@ -2557,10 +2557,10 @@ "expected_e2e_ms": 1042.43, "expected_avg_denoise_ms": 82.26, "expected_median_denoise_ms": 81.75, - "load_peak_vram_mb": 7176.0, - "runtime_peak_vram_mb": 12634.0, - "load_peak_allocated_mb": 7033.0, - "runtime_peak_allocated_mb": 9311.0, + "load_peak_vram_mb": 14558.0, + "runtime_peak_vram_mb": 19230.0, + "load_peak_allocated_mb": 14399.24, + "runtime_peak_allocated_mb": 16295.65, "estimated_full_test_time_s": 40.0 }, "hunyuan3d_shape_gen": { @@ -2700,10 +2700,10 @@ "expected_e2e_ms": 8273.83, "expected_avg_denoise_ms": 146.61, "expected_median_denoise_ms": 147.3, - "load_peak_vram_mb": 7814.0, - "runtime_peak_vram_mb": 18080.0, - "load_peak_allocated_mb": 3641.0, - "runtime_peak_allocated_mb": 9673.0, + "load_peak_vram_mb": 26126.0, + "runtime_peak_vram_mb": 34026.0, + "load_peak_allocated_mb": 25740.95, + "runtime_peak_allocated_mb": 31340.91, "estimated_full_test_time_s": 70 }, "flux_2_image_t2i_upscaling_4x": { @@ -2843,10 +2843,10 @@ "expected_e2e_ms": 8333.66, "expected_avg_denoise_ms": 144.68, "expected_median_denoise_ms": 144.74, - "load_peak_vram_mb": 7814.0, - "runtime_peak_vram_mb": 24380.0, - "load_peak_allocated_mb": 3641.0, - "runtime_peak_allocated_mb": 10508.0, + "load_peak_vram_mb": 26126.0, + "runtime_peak_vram_mb": 39714.0, + "load_peak_allocated_mb": 25740.95, + "runtime_peak_allocated_mb": 31340.91, "estimated_full_test_time_s": 100 }, "wan2_1_t2v_1.3b_frame_interp_2x_upscaling_4x": { @@ -2914,10 +2914,10 @@ "expected_e2e_ms": 8099.03, "expected_avg_denoise_ms": 143.71, "expected_median_denoise_ms": 144.67, - "load_peak_vram_mb": 7814.0, - "runtime_peak_vram_mb": 24388.0, - "load_peak_allocated_mb": 3641.0, - "runtime_peak_allocated_mb": 10531.0, + "load_peak_vram_mb": 26126.0, + "runtime_peak_vram_mb": 40406.0, + "load_peak_allocated_mb": 25740.95, + "runtime_peak_allocated_mb": 31340.91, "estimated_full_test_time_s": 100 }, "ltx_2.3_one_stage_ti2v": { diff --git a/python/sglang/multimodal_gen/test/unit/conftest.py b/python/sglang/multimodal_gen/test/unit/conftest.py index 8f68d0dd9d..53f3f52404 100644 --- a/python/sglang/multimodal_gen/test/unit/conftest.py +++ b/python/sglang/multimodal_gen/test/unit/conftest.py @@ -127,6 +127,11 @@ def mock_cuda_platform( "sglang.multimodal_gen.runtime.platforms.current_platform.enable_dit_layerwise_offload_by_default", return_value=True, ), + # checkpoint sizing reads the local HF cache; tests that need it pass a size + patch( + "sglang.multimodal_gen.runtime.server_args.auto_tune.checkpoint_component_sizes", + return_value=None, + ), ): yield diff --git a/python/sglang/multimodal_gen/test/unit/test_server_args.py b/python/sglang/multimodal_gen/test/unit/test_server_args.py index 506f29aa18..fc50e7f6a3 100644 --- a/python/sglang/multimodal_gen/test/unit/test_server_args.py +++ b/python/sglang/multimodal_gen/test/unit/test_server_args.py @@ -1232,6 +1232,8 @@ class TestOffloadDefaults(_CudaPlatformTestCase): memory_gb=80, available_memory_gb=None, dit_parameter_count=None, + checkpoint_sizes=None, + is_cuda=True, kwargs=None, ): def get_available_gpu_memory(device_id=0, **_kwargs): @@ -1251,6 +1253,10 @@ class TestOffloadDefaults(_CudaPlatformTestCase): else Mock(return_value=dit_parameter_count) ), ), + patch( + "sglang.multimodal_gen.runtime.server_args.auto_tune.checkpoint_component_sizes", + return_value=checkpoint_sizes, + ), patch( "sglang.multimodal_gen.runtime.platforms.current_platform.is_cpu", return_value=False, @@ -1261,7 +1267,7 @@ class TestOffloadDefaults(_CudaPlatformTestCase): ), patch( "sglang.multimodal_gen.runtime.platforms.current_platform.is_cuda", - return_value=True, + return_value=is_cuda, ), patch( "sglang.multimodal_gen.runtime.platforms.current_platform.enable_dit_layerwise_offload_by_default", @@ -2107,6 +2113,101 @@ class TestOffloadDefaults(_CudaPlatformTestCase): ["text_encoder", "image_encoder"], ) + def _zimage_turbo( + self, *, memory_gb=141, available_memory_gb=139, checkpoint_sizes=None, **kwargs + ): + return self._from_dict_with_pipeline_config( + ZImagePipelineConfig(), + memory_gb=memory_gb, + available_memory_gb=available_memory_gb, + checkpoint_sizes=checkpoint_sizes, + kwargs={ + "model_path": "Tongyi-MAI/Z-Image-Turbo", + "performance_mode": "auto", + **kwargs, + }, + ) + + # (parameters, stored bytes) of Tongyi-MAI/Z-Image-Turbo: the DiT is stored in + # fp32 (22.9 GiB) and served in bf16, 19.3 GiB per GPU in all + ZIMAGE_TURBO_SIZES = { + "transformer": (6_154_908_736, 24_619_634_944), + "text_encoder": (4_022_468_096, 8_044_936_192), + "vae": (83_819_683, 335_278_732), + } + + def _residency(self, args): + return { + name: args.residency_mode(name) + for name in ("transformer", "text_encoder", "image_encoder", "vae") + } + + def test_auto_keeps_every_component_resident_when_weights_fit_twice(self): + args = self._zimage_turbo(checkpoint_sizes=self.ZIMAGE_TURBO_SIZES) + + self.assertIsNone(args.layerwise_offload_components) + self.assertFalse(args.dit_cpu_offload) + self.assertFalse(args.text_encoder_cpu_offload) + self.assertEqual(set(self._residency(args).values()), {RESIDENT}) + + def test_auto_tier_sizes_weights_in_their_serving_precision(self): + # fits twice in 45 GiB at bf16; the fp32-stored bytes would not + args = self._zimage_turbo( + memory_gb=48, + available_memory_gb=45, + checkpoint_sizes=self.ZIMAGE_TURBO_SIZES, + ) + self.assertEqual(set(self._residency(args).values()), {RESIDENT}) + + def test_auto_tier_leaves_consumer_gpus_on_their_policy(self): + for memory_gb, available_gb in ((32, 31), (24, 23)): + with self.subTest(memory_gb=memory_gb): + sized = self._zimage_turbo( + memory_gb=memory_gb, + available_memory_gb=available_gb, + checkpoint_sizes=self.ZIMAGE_TURBO_SIZES, + ) + unsized = self._zimage_turbo( + memory_gb=memory_gb, available_memory_gb=available_gb + ) + self.assertEqual(self._residency(sized), self._residency(unsized)) + self.assertNotEqual(sized.residency_mode("text_encoder"), RESIDENT) + + def test_auto_tier_skips_batching_shared_memory_and_unsized_checkpoints(self): + unsized = self._zimage_turbo() + self.assertEqual(unsized.residency_mode("text_encoder"), LAYERWISE_OFFLOAD) + batching = self._zimage_turbo( + checkpoint_sizes=self.ZIMAGE_TURBO_SIZES, batching_max_size=2 + ) + self.assertEqual(batching.residency_mode("text_encoder"), LAYERWISE_OFFLOAD) + with patch.object( + current_platform, "device_shares_host_memory", return_value=True + ): + shared = self._zimage_turbo(checkpoint_sizes=self.ZIMAGE_TURBO_SIZES) + self.assertEqual(shared.residency_mode("text_encoder"), LAYERWISE_OFFLOAD) + + def test_auto_tier_replaces_whole_component_offload_off_cuda(self): + # off CUDA auto has no layerwise defaults and offloads the encoder whole + unsized = self._from_dict_with_pipeline_config( + ZImagePipelineConfig(), + memory_gb=192, + available_memory_gb=190, + is_cuda=False, + kwargs={"model_path": "Tongyi-MAI/Z-Image-Turbo"}, + ) + sized = self._from_dict_with_pipeline_config( + ZImagePipelineConfig(), + memory_gb=192, + available_memory_gb=190, + checkpoint_sizes=self.ZIMAGE_TURBO_SIZES, + is_cuda=False, + kwargs={"model_path": "Tongyi-MAI/Z-Image-Turbo"}, + ) + + self.assertTrue(unsized.text_encoder_cpu_offload) + self.assertFalse(sized.text_encoder_cpu_offload) + self.assertEqual(set(self._residency(sized).values()), {RESIDENT}) + def test_auto_lingbot_keeps_dit_resident_on_h100(self): args = self._from_dict_with_pipeline_config( LingBotWorldCausalDMDConfig(), @@ -2215,6 +2316,45 @@ class TestOffloadDefaults(_CudaPlatformTestCase): ["dit", "text_encoder", "image_encoder", "vae"], ) + def test_auto_wan2_2_a14b_keeps_both_experts_resident_above_120gb(self): + for pipeline_config, model_path in ( + (Wan2_2_T2V_A14B_Config(), "Wan-AI/Wan2.2-T2V-A14B-Diffusers"), + (Wan2_2_I2V_A14B_Config(), "Wan-AI/Wan2.2-I2V-A14B-Diffusers"), + ): + with self.subTest(pipeline_config=pipeline_config.__class__.__name__): + args = self._from_dict_with_pipeline_config( + pipeline_config, + memory_gb=141, + available_memory_gb=139, + kwargs={"model_path": model_path, "performance_mode": "auto"}, + ) + + self.assertFalse(args.dit_cpu_offload) + self.assertEqual(args.residency_mode("transformer"), RESIDENT) + self.assertEqual(args.residency_mode("transformer_2"), RESIDENT) + self.assertEqual(args.residency_mode("vae"), RESIDENT) + self.assertEqual( + args.layerwise_offload_components, + ["text_encoder", "image_encoder"], + ) + + # two 14B experts plus T5 fit twice over: the encoder joins them + rich = self._from_dict_with_pipeline_config( + pipeline_config, + memory_gb=288, + available_memory_gb=280, + checkpoint_sizes={ + "transformer": (14_288_901_184, 57_155_604_736), + "transformer_2": (14_288_901_184, 57_155_604_736), + "text_encoder": (5_680_910_336, 22_723_641_344), + "vae": (126_892_531, 507_570_124), + }, + kwargs={"model_path": model_path, "performance_mode": "auto"}, + ) + self.assertIsNone(rich.layerwise_offload_components) + self.assertEqual(rich.residency_mode("text_encoder"), RESIDENT) + self.assertEqual(rich.residency_mode("transformer_2"), RESIDENT) + def test_auto_wan2_1_14b_keeps_dit_resident_on_h100(self): for pipeline_config, model_path in ( (WanT2V720PConfig(), "Wan-AI/Wan2.1-T2V-14B-Diffusers"), diff --git a/python/sglang/multimodal_gen/test/unit/test_weight_utils.py b/python/sglang/multimodal_gen/test/unit/test_weight_utils.py index 3591f62848..a349e9dbba 100644 --- a/python/sglang/multimodal_gen/test/unit/test_weight_utils.py +++ b/python/sglang/multimodal_gen/test/unit/test_weight_utils.py @@ -2,6 +2,7 @@ """Unit tests for multimodal weight-loading utilities.""" import os +import shutil import tempfile import unittest from pathlib import Path @@ -15,6 +16,7 @@ from safetensors.torch import save_file as safetensors_save_file from sglang.multimodal_gen.runtime.loader.utils import ( _list_safetensors_files, checkpoint_bytes, + checkpoint_component_sizes, dit_parameter_count, ) from sglang.multimodal_gen.runtime.loader.weight_utils import ( @@ -266,6 +268,79 @@ class TestDitParameterCount(CustomTestCase): self.assertEqual(hub_download.called, use_modelscope == "0") +def _write_pipeline(root: Path) -> None: + """A Diffusers layout: a 58-byte DiT, a 16-byte encoder and weightless parts.""" + _write_sharded_transformer(root) + (root / "text_encoder").mkdir() + safetensors_save_file( + {"e": torch.zeros(4)}, root / "text_encoder" / "model.safetensors" + ) + for weightless in ("tokenizer", "scheduler"): + (root / weightless).mkdir() + (root / weightless / "config.json").write_text("{}") + (root / "model_index.json").write_text( + '{"_class_name": "P", "transformer": ["diffusers", "T"], ' + '"text_encoder": ["transformers", "E"], "tokenizer": ["transformers", "K"], ' + '"scheduler": ["diffusers", "S"], "safety_checker": [null, null], ' + '"boundary_ratio": 0.9}' + ) + + +class TestCheckpointComponentSizes(CustomTestCase): + SIZES = {"transformer": (17, 58), "text_encoder": (4, 16)} + + def test_sizes_every_weighted_component(self): + with tempfile.TemporaryDirectory() as tmpdir: + _write_pipeline(Path(tmpdir) / "sub") + + self.assertEqual( + checkpoint_component_sizes(tmpdir, subfolder="sub", revision=None), + self.SIZES, + ) + + def test_cached_hub_snapshot_is_sized_without_the_network(self): + with tempfile.TemporaryDirectory() as tmpdir: + _write_pipeline(Path(tmpdir)) + with patch(_SNAPSHOT_DOWNLOAD, return_value=tmpdir) as snapshot_download: + self.assertEqual( + checkpoint_component_sizes( + "org/model", subfolder=None, revision="abc" + ), + self.SIZES, + ) + self.assertTrue(snapshot_download.call_args.kwargs["local_files_only"]) + + def test_partial_download_other_formats_or_quantized_weights_are_unsized(self): + cases = { + # only model_index.json fetched so far, as before the worker downloads + "index only": lambda root: shutil.rmtree(root / "text_encoder"), + "missing shard": lambda root: ( + root / "transformer" / "shard-2.safetensors" + ).unlink(), + "bin weights": lambda root: ( + (root / "text_encoder" / "model.safetensors").unlink(), + (root / "text_encoder" / "pytorch_model.bin").write_bytes(b"0"), + ), + "quantized": lambda root: safetensors_save_file( + {"q": torch.zeros(4, dtype=torch.float8_e4m3fn)}, + root / "text_encoder" / "model.safetensors", + ), + } + for name, damage in cases.items(): + with self.subTest(name), tempfile.TemporaryDirectory() as tmpdir: + _write_pipeline(Path(tmpdir)) + damage(Path(tmpdir)) + self.assertIsNone( + checkpoint_component_sizes(tmpdir, subfolder=None, revision=None) + ) + + def test_uncached_repo_is_unsized(self): + with patch(_SNAPSHOT_DOWNLOAD, side_effect=LocalEntryNotFoundError("miss")): + self.assertIsNone( + checkpoint_component_sizes("org/model", subfolder=None, revision=None) + ) + + class TestDisableRunaiStreamerRankDiscoveryCollective(unittest.TestCase): def test_never_touches_torch_distributed_even_when_initialized(self): from runai_model_streamer.distributed_streamer.distributed_streamer import (