[Perf] diffusion: Turn on breakable CUDA graphs by default for LTX-2/LTX-2.3 two-stage serving (#45977)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Mick Qian
Claude Opus 5.5
parent
2e6d87e2ca
commit
e66c9e981d
@@ -210,6 +210,7 @@ class LTX2PipelineConfig(PipelineConfig):
|
||||
keep_resident_min_available_gb=70,
|
||||
keep_resident_components=("dit",),
|
||||
auto_cfg_parallel_degree_by_num_gpus=((4, 1), (8, 1)),
|
||||
bcg_default_pipeline_class_names=("LTX2TwoStagePipeline",),
|
||||
)
|
||||
|
||||
def prepare_latent_shape(self, batch, batch_size, num_frames):
|
||||
|
||||
@@ -27,6 +27,9 @@ class ModelDeploymentConfig:
|
||||
# torch.compile is model opt-in because it can be slower than eager for
|
||||
# diffusion workloads dominated by already-optimized kernels
|
||||
speed_mode_enable_torch_compile_by_default: bool = False
|
||||
# pipeline classes measured faster with breakable CUDA graphs; auto and speed
|
||||
# modes enable BCG for them unless a gate in ServerArgs rules it out
|
||||
bcg_default_pipeline_class_names: tuple[str, ...] = ()
|
||||
supports_cfg_parallel: bool = True
|
||||
|
||||
def get_auto_cfg_parallel_degree(self, num_gpus: int) -> int:
|
||||
|
||||
@@ -82,6 +82,7 @@ from sglang.multimodal_gen.runtime.pipelines_core import (
|
||||
build_pipeline,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.denoising import DenoisingStage
|
||||
from sglang.multimodal_gen.runtime.platforms import (
|
||||
current_platform,
|
||||
initialize_current_platform,
|
||||
@@ -1606,6 +1607,7 @@ class GPUWorker(GPUWorkerPostTrainingMixin):
|
||||
)
|
||||
finally:
|
||||
self._update_lora_metrics()
|
||||
self._drop_breakable_cuda_graphs("LoRA changed")
|
||||
return OutputBatch()
|
||||
|
||||
def merge_lora_weights(
|
||||
@@ -1624,6 +1626,7 @@ class GPUWorker(GPUWorkerPostTrainingMixin):
|
||||
self.pipeline.merge_lora_weights(target, strength)
|
||||
finally:
|
||||
self._update_lora_metrics()
|
||||
self._drop_breakable_cuda_graphs("LoRA changed")
|
||||
return OutputBatch()
|
||||
|
||||
def unmerge_lora_weights(self, target: str = "all") -> OutputBatch:
|
||||
@@ -1639,8 +1642,15 @@ class GPUWorker(GPUWorkerPostTrainingMixin):
|
||||
self.pipeline.unmerge_lora_weights(target)
|
||||
finally:
|
||||
self._update_lora_metrics()
|
||||
self._drop_breakable_cuda_graphs("LoRA changed")
|
||||
return OutputBatch()
|
||||
|
||||
def _drop_breakable_cuda_graphs(self, reason: str) -> None:
|
||||
# captured graphs replay the weights and LoRA branches of capture time
|
||||
for stage in self.pipeline.stages:
|
||||
if isinstance(stage, DenoisingStage):
|
||||
stage.drop_breakable_cuda_graphs(reason)
|
||||
|
||||
def _update_lora_metrics(self) -> None:
|
||||
if self.metrics is not None and isinstance(self.pipeline, LoRAPipeline):
|
||||
self.metrics.update_lora(self.pipeline.get_lora_status())
|
||||
@@ -1668,7 +1678,9 @@ class GPUWorker(GPUWorkerPostTrainingMixin):
|
||||
"sleeping": False,
|
||||
"message": "already awake",
|
||||
}
|
||||
return self.memory_occupation.resume_memory_occupation()
|
||||
result = self.memory_occupation.resume_memory_occupation()
|
||||
self._drop_breakable_cuda_graphs("weights moved by resume_memory_occupation")
|
||||
return result
|
||||
|
||||
|
||||
OOM_MSG = """
|
||||
|
||||
@@ -2564,6 +2564,19 @@ class DenoisingStage(PipelineStage):
|
||||
return padder(call_kwargs, current_model, buckets)
|
||||
return bcg_utils.pad_masked_prompt_kwargs(call_kwargs, buckets)
|
||||
|
||||
def drop_breakable_cuda_graphs(self, reason: str) -> None:
|
||||
"""Disable graphs whose weights changed after capture; serving never re-captures."""
|
||||
dropped = sum(len(runner.entries) for runner in self._bcg_runners.values())
|
||||
for runner in self._bcg_runners.values():
|
||||
runner.reset(disabled_reason=reason)
|
||||
if dropped:
|
||||
logger.warning(
|
||||
"[Diffusion BCG] %s after capture: dropped %d graph(s); "
|
||||
"requests run eager until restart.",
|
||||
reason,
|
||||
dropped,
|
||||
)
|
||||
|
||||
def _maybe_get_bcg_runner(self, current_model):
|
||||
"""Return (lazily creating) the breakable CUDA graph runner for
|
||||
``current_model``, or ``None`` if BCG is disabled / inapplicable.
|
||||
|
||||
@@ -511,10 +511,10 @@ class ServerArgs(DisaggServerArgsMixin):
|
||||
# values so the exact tier's speed is unchanged.
|
||||
allow_cudnn_tf32: bool = True
|
||||
allow_bf16_reduced_precision_reduction: bool = True
|
||||
# BCG graphs are resolution-specific, so --warmup-resolutions is required
|
||||
# when BCG is enabled: every requested resolution is captured at warmup so
|
||||
# serving never triggers a fresh capture.
|
||||
enable_breakable_cuda_graph: bool = False
|
||||
# BCG graphs are resolution-specific: every warmup resolution is captured at
|
||||
# warmup so serving never triggers a fresh capture. None resolves in
|
||||
# _adjust_breakable_cuda_graph_support (on for opted-in pipelines).
|
||||
enable_breakable_cuda_graph: bool | None = None
|
||||
# Text/prompt sequence-length padding budget for BCG. Prompt-conditioning
|
||||
# inputs are padded up to the smallest bucket that fits, so prompts of
|
||||
# different lengths reuse one captured graph. Warmup captures one graph per
|
||||
@@ -818,6 +818,8 @@ class ServerArgs(DisaggServerArgsMixin):
|
||||
return None
|
||||
|
||||
def _adjust_breakable_cuda_graph_support(self):
|
||||
if self.enable_breakable_cuda_graph is None:
|
||||
self.enable_breakable_cuda_graph = self._breakable_cuda_graph_default()
|
||||
if not self.enable_breakable_cuda_graph:
|
||||
return
|
||||
|
||||
@@ -852,6 +854,52 @@ class ServerArgs(DisaggServerArgsMixin):
|
||||
if not self.warmup_resolutions and not self.comfyui_mode:
|
||||
self._default_bcg_warmup_resolution()
|
||||
|
||||
def _breakable_cuda_graph_default(self) -> bool:
|
||||
"""Whether unset BCG turns on: an opted-in pipeline on a validated checkpoint,
|
||||
captured by the startup warmup over DiT weights that never move."""
|
||||
deployment_config = self.pipeline_config.get_model_deployment_config()
|
||||
if (
|
||||
self.pipeline_class_name
|
||||
not in deployment_config.bcg_default_pipeline_class_names
|
||||
):
|
||||
return False
|
||||
enabled = (
|
||||
self.performance_mode in ("auto", "speed")
|
||||
and current_platform.is_cuda()
|
||||
and not current_platform.device_shares_host_memory()
|
||||
# offline runs have no startup warmup; capturing would add a generation
|
||||
and self.warmup_mode == "server"
|
||||
and self.disagg_role == RoleType.MONOLITHIC
|
||||
and not self.comfyui_mode
|
||||
and self._is_breakable_cuda_graph_supported_model()
|
||||
and not self.enable_torch_compile
|
||||
and not envs.SGLANG_CACHE_DIT_ENABLED
|
||||
and self.batching_max_size == 1
|
||||
and self.lora_path is None
|
||||
# measured with both stage DiTs resident on >=130 GiB cards; capture
|
||||
# holds ~7 GB more on Hopper, past an 80 GB card's speed-mode headroom
|
||||
and current_platform.get_device_total_memory() / BYTES_PER_GB
|
||||
>= LTX2_RESIDENT_AUTO_ENABLE_MEM_GB
|
||||
and (
|
||||
self.ltx2_two_stage_device_mode == "resident"
|
||||
or not self._is_ltx23_two_stage_pipeline()
|
||||
)
|
||||
# graphs replay fixed weight addresses
|
||||
and not self.dit_layerwise_offload
|
||||
and not self.use_fsdp_inference
|
||||
and all(
|
||||
self.residency_mode(name) == RESIDENT
|
||||
for name in ("transformer", "transformer_2")
|
||||
)
|
||||
)
|
||||
if enabled:
|
||||
logger.info(
|
||||
"[Diffusion BCG] enabled by default for %s; pass "
|
||||
"--enable-breakable-cuda-graph false to run eager.",
|
||||
self.pipeline_class_name,
|
||||
)
|
||||
return enabled
|
||||
|
||||
def _is_breakable_cuda_graph_supported_model(self) -> bool:
|
||||
refs = _normalized_bcg_model_refs(self.model_id)
|
||||
refs.update(_normalized_bcg_model_refs(self.model_path))
|
||||
@@ -2595,7 +2643,9 @@ class ServerArgs(DisaggServerArgsMixin):
|
||||
"(split at attention; SP all-to-all / dynamic attention stay "
|
||||
"eager) to cut per-kernel launch overhead. Mutually exclusive "
|
||||
"with --enable-torch-compile and Cache-DiT (BCG takes priority). "
|
||||
"Requires --warmup-resolutions; all of them are captured at warmup.",
|
||||
"Graphs are captured at warmup for --warmup-resolutions (default: "
|
||||
"the model default shape). Unset, it is on for LTX-2/LTX-2.3 "
|
||||
"LTX2TwoStagePipeline serving with resident DiTs on >=130 GiB GPUs.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--bcg-text-buckets",
|
||||
|
||||
@@ -128,10 +128,15 @@ def _resolve_default_warmup_resolution(
|
||||
width = sampling_defaults.width
|
||||
height = sampling_defaults.height
|
||||
is_image_gen = server_args.pipeline_config.task_type.is_image_gen()
|
||||
# graphs replay only the captured shape, so BCG skips the video area cap
|
||||
if (
|
||||
width is not None
|
||||
and height is not None
|
||||
and (not server_based_warmup or is_image_gen)
|
||||
and (
|
||||
not server_based_warmup
|
||||
or is_image_gen
|
||||
or server_args.enable_breakable_cuda_graph is True
|
||||
)
|
||||
):
|
||||
return width, height
|
||||
|
||||
|
||||
@@ -66,6 +66,7 @@ from sglang.multimodal_gen.runtime.warmup_request_builder import (
|
||||
DEFAULT_PLACEHOLDER_PROMPT,
|
||||
SERVER_WARMUP_IMAGE_FALLBACK_RESOLUTION,
|
||||
_apply_warmup_sampling_overrides,
|
||||
_resolve_default_warmup_resolution,
|
||||
_resolve_warmup_num_frames,
|
||||
build_warmup_reqs,
|
||||
should_include_warmup_image,
|
||||
@@ -834,6 +835,25 @@ class TestWarmupReqCfgParallel(unittest.TestCase):
|
||||
self.assertEqual(reqs[0].num_frames, 17)
|
||||
self.assertEqual(reqs[0].num_inference_steps, 2)
|
||||
|
||||
def test_bcg_capture_keeps_the_full_default_video_resolution(self):
|
||||
# graphs replay only the captured shape, so the video area cap must not apply
|
||||
server_args = MagicMock()
|
||||
server_args.pipeline_config.task_type = ModelTaskType.T2V
|
||||
server_args.enable_breakable_cuda_graph = True
|
||||
sampling_defaults = SamplingParams(
|
||||
width=1280,
|
||||
height=720,
|
||||
num_frames=81,
|
||||
supported_resolutions=[(1280, 720), (832, 480)],
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
_resolve_default_warmup_resolution(
|
||||
server_args, sampling_defaults, server_based_warmup=True
|
||||
),
|
||||
(1280, 720),
|
||||
)
|
||||
|
||||
def test_ltx2_two_stage_warmup_uses_pipeline_alignment(self):
|
||||
server_args = MagicMock()
|
||||
server_args.warmup_steps = 1
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import dataclasses
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import torch
|
||||
|
||||
@@ -19,10 +19,12 @@ from sglang.multimodal_gen.runtime.layers.attention import (
|
||||
DynamicVarlenMaskMeta,
|
||||
build_varlen_mask_meta,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.gpu_worker import GPUWorker
|
||||
from sglang.multimodal_gen.runtime.models.dits.flux3_image import Packed
|
||||
from sglang.multimodal_gen.runtime.models.dits.qwen_image import (
|
||||
_attn_mask_meta_local_pad,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core import LoRAPipeline
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages import (
|
||||
denoising as denoising_module,
|
||||
)
|
||||
@@ -898,5 +900,64 @@ class TestDiffusionBCGPadding(unittest.TestCase):
|
||||
self.assertIsNone(runner._capture_limit_reason(entry))
|
||||
|
||||
|
||||
class TestBCGWeightChangeAfterCapture(unittest.TestCase):
|
||||
"""Graphs replay capture-time weights and LoRA branches, so a later change drops them."""
|
||||
|
||||
def setUp(self):
|
||||
self.eager_calls = 0
|
||||
|
||||
def eager(**kwargs):
|
||||
self.eager_calls += 1
|
||||
return "eager"
|
||||
|
||||
self.runner = object.__new__(DiffusionBreakableCudaGraphRunner)
|
||||
self.runner.transformer = eager
|
||||
self.runner.device_module = SimpleNamespace(empty_cache=lambda: None)
|
||||
self.runner._blocked = set()
|
||||
self.runner._disabled_reason = None
|
||||
self.kwargs = {"hidden_states": torch.zeros(2)}
|
||||
self.runner.entries = {
|
||||
_signature_kwargs(self.kwargs): _CaptureEntry(
|
||||
graph=SimpleNamespace(_break_fns=[], _segments=[object()]),
|
||||
static_kwargs={},
|
||||
static_leaves=[],
|
||||
output=None,
|
||||
num_segments=1,
|
||||
)
|
||||
}
|
||||
stage = DenoisingStage.__new__(DenoisingStage)
|
||||
stage._bcg_runners = {(id(eager), "lossless"): self.runner}
|
||||
self.worker = GPUWorker.__new__(GPUWorker)
|
||||
self.worker.metrics = None
|
||||
self.worker.pipeline = MagicMock(spec=LoRAPipeline)
|
||||
self.worker.pipeline.stages = [stage]
|
||||
|
||||
def _assert_runs_eager(self):
|
||||
self.assertEqual(self.runner.entries, {})
|
||||
self.assertEqual(self.runner(**self.kwargs), "eager")
|
||||
self.assertEqual(self.eager_calls, 1)
|
||||
|
||||
def test_set_lora_drops_captured_graphs(self):
|
||||
self.worker.set_lora("style", "/loras/style.safetensors")
|
||||
self.worker.pipeline.set_lora.assert_called_once()
|
||||
self._assert_runs_eager()
|
||||
|
||||
def test_merge_and_unmerge_drop_captured_graphs(self):
|
||||
changes = {
|
||||
"merge": lambda worker: worker.merge_lora_weights(),
|
||||
"unmerge": lambda worker: worker.unmerge_lora_weights(),
|
||||
}
|
||||
for name, change in changes.items():
|
||||
with self.subTest(name):
|
||||
self.setUp()
|
||||
change(self.worker)
|
||||
self._assert_runs_eager()
|
||||
|
||||
def test_resume_memory_occupation_drops_captured_graphs(self):
|
||||
self.worker.memory_occupation = MagicMock()
|
||||
self.worker.resume_memory_occupation()
|
||||
self._assert_runs_eager()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -34,6 +34,7 @@ def _make_layer() -> BaseLayerWithLoRA:
|
||||
|
||||
def _make_pipeline(layer: BaseLayerWithLoRA) -> _TestLoRAPipeline:
|
||||
pipeline = object.__new__(_TestLoRAPipeline)
|
||||
pipeline._stages = []
|
||||
pipeline.modules = {"transformer": torch.nn.Module()}
|
||||
pipeline.server_args = SimpleNamespace(
|
||||
lora_alpha=None,
|
||||
|
||||
@@ -3164,6 +3164,110 @@ class TestOffloadDefaults(_CudaPlatformTestCase):
|
||||
parser.parse_known_args(argv)
|
||||
|
||||
|
||||
class TestBreakableCudaGraphDefault(_CudaPlatformTestCase):
|
||||
"""Unset BCG turns on only where the served path keeps fixed DiT weights."""
|
||||
|
||||
LTX2_TWO_STAGE_SERVE = {
|
||||
"model_path": "Lightricks/LTX-2",
|
||||
"num_gpus": 2,
|
||||
"pipeline_class_name": "LTX2TwoStagePipeline",
|
||||
"enable_cfg_parallel": True,
|
||||
"warmup_mode": "server",
|
||||
}
|
||||
|
||||
def _resolve(
|
||||
self,
|
||||
*,
|
||||
memory_gb=141,
|
||||
available_memory_gb=139,
|
||||
device_name="NVIDIA H200",
|
||||
**overrides,
|
||||
):
|
||||
with (
|
||||
patch.object(
|
||||
current_platform, "device_shares_host_memory", return_value=False
|
||||
),
|
||||
patch.object(current_platform, "get_device_name", return_value=device_name),
|
||||
):
|
||||
return TestOffloadDefaults._from_dict_with_pipeline_config(
|
||||
self,
|
||||
LTX2PipelineConfig(),
|
||||
memory_gb=memory_gb,
|
||||
available_memory_gb=available_memory_gb,
|
||||
kwargs={**self.LTX2_TWO_STAGE_SERVE, **overrides},
|
||||
)
|
||||
|
||||
def test_flagless_ltx2_two_stage_serving_captures_the_model_default(self):
|
||||
args = self._resolve()
|
||||
self.assertTrue(args.enable_breakable_cuda_graph)
|
||||
self.assertEqual(args.warmup_resolutions, ["768x512"])
|
||||
self.assertEqual(args.warmup_mode, "server")
|
||||
|
||||
def test_ltx23_needs_both_stage_dits_resident(self):
|
||||
resident = self._resolve(model_path="Lightricks/LTX-2.3")
|
||||
self.assertEqual(resident.ltx2_two_stage_device_mode, "resident")
|
||||
self.assertTrue(resident.enable_breakable_cuda_graph)
|
||||
|
||||
original = self._resolve(
|
||||
model_path="Lightricks/LTX-2.3", ltx2_two_stage_device_mode="original"
|
||||
)
|
||||
self.assertFalse(original.enable_breakable_cuda_graph)
|
||||
|
||||
def test_gates_keep_unset_bcg_off(self):
|
||||
cases = {
|
||||
"explicit off": {"enable_breakable_cuda_graph": False},
|
||||
"offline run": {"warmup_mode": None},
|
||||
"request warmup": {"warmup_mode": "request"},
|
||||
"memory mode": {"performance_mode": "memory"},
|
||||
"manual mode": {"performance_mode": "manual"},
|
||||
"torch.compile": {"enable_torch_compile": True},
|
||||
"dynamic batching": {"batching_max_size": 2},
|
||||
"startup LoRA": {"lora_path": "/loras/style.safetensors"},
|
||||
"ComfyUI": {"comfyui_mode": True},
|
||||
"one-stage pipeline": {"pipeline_class_name": "LTX2Pipeline"},
|
||||
"HQ pipeline": {"pipeline_class_name": "LTX2TwoStageHQPipeline"},
|
||||
"unvalidated checkpoint": {"model_path": "/models/custom-ltx"},
|
||||
"DiT layerwise offload": {"dit_layerwise_offload": True},
|
||||
"DiT component offload": {"dit_cpu_offload": True},
|
||||
"DiT residency selector": {
|
||||
"component_residency": ["transformer=layerwise-offload"]
|
||||
},
|
||||
# below LTX-2's keep-resident threshold auto component-offloads the DiT
|
||||
"low free memory": {"available_memory_gb": 60},
|
||||
# the DiT stays resident, but capture outgrows a speed-mode 80 GB card
|
||||
"80 GB card": {
|
||||
"memory_gb": 80,
|
||||
"available_memory_gb": 79,
|
||||
"device_name": "NVIDIA H100 80GB HBM3",
|
||||
},
|
||||
}
|
||||
for name, overrides in cases.items():
|
||||
with self.subTest(name):
|
||||
self.assertFalse(self._resolve(**overrides).enable_breakable_cuda_graph)
|
||||
|
||||
def test_platform_gates(self):
|
||||
args = self._resolve(enable_breakable_cuda_graph=False)
|
||||
with patch.object(
|
||||
current_platform, "get_device_total_memory", return_value=141 * 1024**3
|
||||
):
|
||||
with patch.object(
|
||||
current_platform, "device_shares_host_memory", return_value=False
|
||||
):
|
||||
self.assertTrue(args._breakable_cuda_graph_default())
|
||||
# ROCm answers torch.cuda.is_available() but not the platform check
|
||||
with patch.object(current_platform, "is_cuda", return_value=False):
|
||||
self.assertFalse(args._breakable_cuda_graph_default())
|
||||
with patch.object(
|
||||
current_platform, "device_shares_host_memory", return_value=True
|
||||
):
|
||||
self.assertFalse(args._breakable_cuda_graph_default())
|
||||
|
||||
def test_explicit_enable_elsewhere_is_unchanged(self):
|
||||
args = self._resolve(enable_breakable_cuda_graph=True, warmup_mode=None)
|
||||
self.assertTrue(args.enable_breakable_cuda_graph)
|
||||
self.assertEqual(args.warmup_mode, "server")
|
||||
|
||||
|
||||
class TestKVGatherDegree(unittest.TestCase):
|
||||
def test_sp2_defaults_to_kv_gather(self):
|
||||
args = _from_dict_without_model_resolution(
|
||||
|
||||
Reference in New Issue
Block a user