diff --git a/docs/cookbook/diffusion/LongLive/LongLive-2.0.mdx b/docs/cookbook/diffusion/LongLive/LongLive-2.0.mdx index c1bdfc424..4da75d45c 100644 --- a/docs/cookbook/diffusion/LongLive/LongLive-2.0.mdx +++ b/docs/cookbook/diffusion/LongLive/LongLive-2.0.mdx @@ -25,6 +25,11 @@ Please refer to the [official SGLang-diffusion installation guide](/docs/sglang- sglang serve --model-path Rabinovich/LongLive-2.0-5B-Diffusers ``` +In `auto` mode, GPUs with at least 60 GiB available keep the DiT, text encoder, +and VAE resident. This uses about 44 GiB on H200 for the 832x480 preset and +avoids moving encoder and decoder layers from host memory on every request. +Smaller GPUs retain the layerwise-offload defaults. + If the GPU runs out of memory, move the text encoder, VAE, and DiT to CPU between stages: ```bash Command diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/longlive2.py b/python/sglang/multimodal_gen/configs/pipeline_configs/longlive2.py index dcd23479f..209a484ab 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/longlive2.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/longlive2.py @@ -6,6 +6,9 @@ from dataclasses import dataclass, field from sglang.multimodal_gen.configs.models import DiTConfig from sglang.multimodal_gen.configs.models.dits.longlive2 import LongLive2VideoConfig from sglang.multimodal_gen.configs.pipeline_configs.base import ModelTaskType +from sglang.multimodal_gen.configs.pipeline_configs.model_deployment_config import ( + ModelDeploymentConfig, +) from sglang.multimodal_gen.configs.pipeline_configs.wan import Wan2_2_TI2V_5B_Config from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger @@ -28,6 +31,13 @@ class LongLive2T2VConfig(Wan2_2_TI2V_5B_Config): dit_config: DiTConfig = field(default_factory=LongLive2VideoConfig) + def get_model_deployment_config(self) -> ModelDeploymentConfig: + return ModelDeploymentConfig( + dit_layerwise_offload_modes=("memory",), + keep_resident_min_available_gb=60, + keep_resident_components=("dit", "text_encoder", "vae"), + ) + def adjust_num_frames(self, num_frames: int) -> int: num_frames = super().adjust_num_frames(num_frames) vae_scale_factor_temporal = self.vae_config.arch_config.scale_factor_temporal diff --git a/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json b/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json index 60cc7373b..634f9c186 100644 --- a/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json +++ b/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json @@ -2713,8 +2713,8 @@ "expected_e2e_ms": 5648.49, "expected_avg_denoise_ms": 477.49, "expected_median_denoise_ms": 56.74, - "load_peak_vram_mb": 16110.0, - "runtime_peak_vram_mb": 45756.0, + "load_peak_vram_mb": 32892.0, + "runtime_peak_vram_mb": 61190.0, "estimated_full_test_time_s": 153.1 }, "longlive2_i2v": { @@ -2731,8 +2731,8 @@ "expected_e2e_ms": 9086.81, "expected_avg_denoise_ms": 492.02, "expected_median_denoise_ms": 151.9, - "load_peak_vram_mb": 16110.0, - "runtime_peak_vram_mb": 45756.0, + "load_peak_vram_mb": 32892.0, + "runtime_peak_vram_mb": 62510.0, "estimated_full_test_time_s": 149.4 }, "lingbot_video_moe_t2v": { diff --git a/python/sglang/multimodal_gen/test/unit/test_server_args.py b/python/sglang/multimodal_gen/test/unit/test_server_args.py index bce674ba3..f40b9fa70 100644 --- a/python/sglang/multimodal_gen/test/unit/test_server_args.py +++ b/python/sglang/multimodal_gen/test/unit/test_server_args.py @@ -24,6 +24,9 @@ from sglang.multimodal_gen.configs.pipeline_configs.lingbot_world import ( from sglang.multimodal_gen.configs.pipeline_configs.longcat_image import ( LongCatImagePipelineConfig, ) +from sglang.multimodal_gen.configs.pipeline_configs.longlive2 import ( + LongLive2T2VConfig, +) from sglang.multimodal_gen.configs.pipeline_configs.ltx_2 import ( LTX2PipelineConfig, LTX23PipelineConfig, @@ -1417,6 +1420,7 @@ class TestOffloadDefaults(unittest.TestCase): lingbot_deployment = LingBotWorldCausalDMDConfig().get_model_deployment_config() ltx_deployment = LTX2PipelineConfig().get_model_deployment_config() ltx23_config = LTX23PipelineConfig() + longlive_deployment = LongLive2T2VConfig().get_model_deployment_config() sana_wm_deployment = SanaWMPipelineConfig().get_model_deployment_config() self.assertIsNone(qwen_deployment.fsdp_auto_min_available_memory_gb) @@ -1454,6 +1458,11 @@ class TestOffloadDefaults(unittest.TestCase): self.assertEqual(ltx_deployment.get_auto_cfg_parallel_degree(4), 1) self.assertEqual(ltx_deployment.get_auto_cfg_parallel_degree(8), 1) self.assertEqual(ltx_deployment.get_auto_cfg_parallel_degree(2), 2) + self.assertEqual(longlive_deployment.keep_resident_min_available_gb, 60) + self.assertEqual( + longlive_deployment.keep_resident_components, + ("dit", "text_encoder", "vae"), + ) self.assertFalse( LTX2PipelineConfig().dit_config.arch_config.enable_packed_qkv_input_a2a ) @@ -1486,6 +1495,25 @@ class TestOffloadDefaults(unittest.TestCase): self.assertEqual(qwen_deployment.keep_resident_components, ("vae",)) self.assertIsNone(qwen_deployment.keep_resident_min_available_gb) + def test_longlive_residency_scales_with_available_memory(self): + high_memory_args = self._from_dict_with_pipeline_config( + LongLive2T2VConfig(), + memory_gb=80, + kwargs={"performance_mode": "auto"}, + ) + high_memory_offload = high_memory_args.layerwise_offload_components or [] + self.assertNotIn("text_encoder", high_memory_offload) + self.assertNotIn("vae", high_memory_offload) + + constrained_args = self._from_dict_with_pipeline_config( + LongLive2T2VConfig(), + memory_gb=50, + kwargs={"performance_mode": "auto"}, + ) + constrained_offload = constrained_args.layerwise_offload_components or [] + self.assertIn("text_encoder", constrained_offload) + self.assertIn("vae", constrained_offload) + def test_qwen_ar_generation_residency_scales_with_available_memory(self): pipeline_configs = ( QwenImageLayeredPipelineConfig(),