[Perf] diffusion: Turn on breakable CUDA graphs by default for LTX-2/LTX-2.3 two-stage serving (#45977)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Mick
2026-10-24 09:22:06 +08:00
committed by GitHub
co-authored by Mick Qian Claude Opus 5.5
parent 2e6d87e2ca
commit e66c9e981d
12 changed files with 281 additions and 9 deletions
@@ -210,6 +210,7 @@ class LTX2PipelineConfig(PipelineConfig):
keep_resident_min_available_gb=70,
keep_resident_components=("dit",),
auto_cfg_parallel_degree_by_num_gpus=((4, 1), (8, 1)),
bcg_default_pipeline_class_names=("LTX2TwoStagePipeline",),
)
def prepare_latent_shape(self, batch, batch_size, num_frames):
@@ -27,6 +27,9 @@ class ModelDeploymentConfig:
# torch.compile is model opt-in because it can be slower than eager for
# diffusion workloads dominated by already-optimized kernels
speed_mode_enable_torch_compile_by_default: bool = False
# pipeline classes measured faster with breakable CUDA graphs; auto and speed
# modes enable BCG for them unless a gate in ServerArgs rules it out
bcg_default_pipeline_class_names: tuple[str, ...] = ()
supports_cfg_parallel: bool = True
def get_auto_cfg_parallel_degree(self, num_gpus: int) -> int:
@@ -82,6 +82,7 @@ from sglang.multimodal_gen.runtime.pipelines_core import (
build_pipeline,
)
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch
from sglang.multimodal_gen.runtime.pipelines_core.stages.denoising import DenoisingStage
from sglang.multimodal_gen.runtime.platforms import (
current_platform,
initialize_current_platform,
@@ -1606,6 +1607,7 @@ class GPUWorker(GPUWorkerPostTrainingMixin):
)
finally:
self._update_lora_metrics()
self._drop_breakable_cuda_graphs("LoRA changed")
return OutputBatch()
def merge_lora_weights(
@@ -1624,6 +1626,7 @@ class GPUWorker(GPUWorkerPostTrainingMixin):
self.pipeline.merge_lora_weights(target, strength)
finally:
self._update_lora_metrics()
self._drop_breakable_cuda_graphs("LoRA changed")
return OutputBatch()
def unmerge_lora_weights(self, target: str = "all") -> OutputBatch:
@@ -1639,8 +1642,15 @@ class GPUWorker(GPUWorkerPostTrainingMixin):
self.pipeline.unmerge_lora_weights(target)
finally:
self._update_lora_metrics()
self._drop_breakable_cuda_graphs("LoRA changed")
return OutputBatch()
def _drop_breakable_cuda_graphs(self, reason: str) -> None:
# captured graphs replay the weights and LoRA branches of capture time
for stage in self.pipeline.stages:
if isinstance(stage, DenoisingStage):
stage.drop_breakable_cuda_graphs(reason)
def _update_lora_metrics(self) -> None:
if self.metrics is not None and isinstance(self.pipeline, LoRAPipeline):
self.metrics.update_lora(self.pipeline.get_lora_status())
@@ -1668,7 +1678,9 @@ class GPUWorker(GPUWorkerPostTrainingMixin):
"sleeping": False,
"message": "already awake",
}
return self.memory_occupation.resume_memory_occupation()
result = self.memory_occupation.resume_memory_occupation()
self._drop_breakable_cuda_graphs("weights moved by resume_memory_occupation")
return result
OOM_MSG = """
@@ -2564,6 +2564,19 @@ class DenoisingStage(PipelineStage):
return padder(call_kwargs, current_model, buckets)
return bcg_utils.pad_masked_prompt_kwargs(call_kwargs, buckets)
def drop_breakable_cuda_graphs(self, reason: str) -> None:
"""Disable graphs whose weights changed after capture; serving never re-captures."""
dropped = sum(len(runner.entries) for runner in self._bcg_runners.values())
for runner in self._bcg_runners.values():
runner.reset(disabled_reason=reason)
if dropped:
logger.warning(
"[Diffusion BCG] %s after capture: dropped %d graph(s); "
"requests run eager until restart.",
reason,
dropped,
)
def _maybe_get_bcg_runner(self, current_model):
"""Return (lazily creating) the breakable CUDA graph runner for
``current_model``, or ``None`` if BCG is disabled / inapplicable.
@@ -511,10 +511,10 @@ class ServerArgs(DisaggServerArgsMixin):
# values so the exact tier's speed is unchanged.
allow_cudnn_tf32: bool = True
allow_bf16_reduced_precision_reduction: bool = True
# BCG graphs are resolution-specific, so --warmup-resolutions is required
# when BCG is enabled: every requested resolution is captured at warmup so
# serving never triggers a fresh capture.
enable_breakable_cuda_graph: bool = False
# BCG graphs are resolution-specific: every warmup resolution is captured at
# warmup so serving never triggers a fresh capture. None resolves in
# _adjust_breakable_cuda_graph_support (on for opted-in pipelines).
enable_breakable_cuda_graph: bool | None = None
# Text/prompt sequence-length padding budget for BCG. Prompt-conditioning
# inputs are padded up to the smallest bucket that fits, so prompts of
# different lengths reuse one captured graph. Warmup captures one graph per
@@ -818,6 +818,8 @@ class ServerArgs(DisaggServerArgsMixin):
return None
def _adjust_breakable_cuda_graph_support(self):
if self.enable_breakable_cuda_graph is None:
self.enable_breakable_cuda_graph = self._breakable_cuda_graph_default()
if not self.enable_breakable_cuda_graph:
return
@@ -852,6 +854,52 @@ class ServerArgs(DisaggServerArgsMixin):
if not self.warmup_resolutions and not self.comfyui_mode:
self._default_bcg_warmup_resolution()
def _breakable_cuda_graph_default(self) -> bool:
"""Whether unset BCG turns on: an opted-in pipeline on a validated checkpoint,
captured by the startup warmup over DiT weights that never move."""
deployment_config = self.pipeline_config.get_model_deployment_config()
if (
self.pipeline_class_name
not in deployment_config.bcg_default_pipeline_class_names
):
return False
enabled = (
self.performance_mode in ("auto", "speed")
and current_platform.is_cuda()
and not current_platform.device_shares_host_memory()
# offline runs have no startup warmup; capturing would add a generation
and self.warmup_mode == "server"
and self.disagg_role == RoleType.MONOLITHIC
and not self.comfyui_mode
and self._is_breakable_cuda_graph_supported_model()
and not self.enable_torch_compile
and not envs.SGLANG_CACHE_DIT_ENABLED
and self.batching_max_size == 1
and self.lora_path is None
# measured with both stage DiTs resident on >=130 GiB cards; capture
# holds ~7 GB more on Hopper, past an 80 GB card's speed-mode headroom
and current_platform.get_device_total_memory() / BYTES_PER_GB
>= LTX2_RESIDENT_AUTO_ENABLE_MEM_GB
and (
self.ltx2_two_stage_device_mode == "resident"
or not self._is_ltx23_two_stage_pipeline()
)
# graphs replay fixed weight addresses
and not self.dit_layerwise_offload
and not self.use_fsdp_inference
and all(
self.residency_mode(name) == RESIDENT
for name in ("transformer", "transformer_2")
)
)
if enabled:
logger.info(
"[Diffusion BCG] enabled by default for %s; pass "
"--enable-breakable-cuda-graph false to run eager.",
self.pipeline_class_name,
)
return enabled
def _is_breakable_cuda_graph_supported_model(self) -> bool:
refs = _normalized_bcg_model_refs(self.model_id)
refs.update(_normalized_bcg_model_refs(self.model_path))
@@ -2595,7 +2643,9 @@ class ServerArgs(DisaggServerArgsMixin):
"(split at attention; SP all-to-all / dynamic attention stay "
"eager) to cut per-kernel launch overhead. Mutually exclusive "
"with --enable-torch-compile and Cache-DiT (BCG takes priority). "
"Requires --warmup-resolutions; all of them are captured at warmup.",
"Graphs are captured at warmup for --warmup-resolutions (default: "
"the model default shape). Unset, it is on for LTX-2/LTX-2.3 "
"LTX2TwoStagePipeline serving with resident DiTs on >=130 GiB GPUs.",
)
parser.add_argument(
"--bcg-text-buckets",
@@ -128,10 +128,15 @@ def _resolve_default_warmup_resolution(
width = sampling_defaults.width
height = sampling_defaults.height
is_image_gen = server_args.pipeline_config.task_type.is_image_gen()
# graphs replay only the captured shape, so BCG skips the video area cap
if (
width is not None
and height is not None
and (not server_based_warmup or is_image_gen)
and (
not server_based_warmup
or is_image_gen
or server_args.enable_breakable_cuda_graph is True
)
):
return width, height
@@ -66,6 +66,7 @@ from sglang.multimodal_gen.runtime.warmup_request_builder import (
DEFAULT_PLACEHOLDER_PROMPT,
SERVER_WARMUP_IMAGE_FALLBACK_RESOLUTION,
_apply_warmup_sampling_overrides,
_resolve_default_warmup_resolution,
_resolve_warmup_num_frames,
build_warmup_reqs,
should_include_warmup_image,
@@ -834,6 +835,25 @@ class TestWarmupReqCfgParallel(unittest.TestCase):
self.assertEqual(reqs[0].num_frames, 17)
self.assertEqual(reqs[0].num_inference_steps, 2)
def test_bcg_capture_keeps_the_full_default_video_resolution(self):
# graphs replay only the captured shape, so the video area cap must not apply
server_args = MagicMock()
server_args.pipeline_config.task_type = ModelTaskType.T2V
server_args.enable_breakable_cuda_graph = True
sampling_defaults = SamplingParams(
width=1280,
height=720,
num_frames=81,
supported_resolutions=[(1280, 720), (832, 480)],
)
self.assertEqual(
_resolve_default_warmup_resolution(
server_args, sampling_defaults, server_based_warmup=True
),
(1280, 720),
)
def test_ltx2_two_stage_warmup_uses_pipeline_alignment(self):
server_args = MagicMock()
server_args.warmup_steps = 1
@@ -1,7 +1,7 @@
import dataclasses
import unittest
from types import SimpleNamespace
from unittest.mock import patch
from unittest.mock import MagicMock, patch
import torch
@@ -19,10 +19,12 @@ from sglang.multimodal_gen.runtime.layers.attention import (
DynamicVarlenMaskMeta,
build_varlen_mask_meta,
)
from sglang.multimodal_gen.runtime.managers.gpu_worker import GPUWorker
from sglang.multimodal_gen.runtime.models.dits.flux3_image import Packed
from sglang.multimodal_gen.runtime.models.dits.qwen_image import (
_attn_mask_meta_local_pad,
)
from sglang.multimodal_gen.runtime.pipelines_core import LoRAPipeline
from sglang.multimodal_gen.runtime.pipelines_core.stages import (
denoising as denoising_module,
)
@@ -898,5 +900,64 @@ class TestDiffusionBCGPadding(unittest.TestCase):
self.assertIsNone(runner._capture_limit_reason(entry))
class TestBCGWeightChangeAfterCapture(unittest.TestCase):
"""Graphs replay capture-time weights and LoRA branches, so a later change drops them."""
def setUp(self):
self.eager_calls = 0
def eager(**kwargs):
self.eager_calls += 1
return "eager"
self.runner = object.__new__(DiffusionBreakableCudaGraphRunner)
self.runner.transformer = eager
self.runner.device_module = SimpleNamespace(empty_cache=lambda: None)
self.runner._blocked = set()
self.runner._disabled_reason = None
self.kwargs = {"hidden_states": torch.zeros(2)}
self.runner.entries = {
_signature_kwargs(self.kwargs): _CaptureEntry(
graph=SimpleNamespace(_break_fns=[], _segments=[object()]),
static_kwargs={},
static_leaves=[],
output=None,
num_segments=1,
)
}
stage = DenoisingStage.__new__(DenoisingStage)
stage._bcg_runners = {(id(eager), "lossless"): self.runner}
self.worker = GPUWorker.__new__(GPUWorker)
self.worker.metrics = None
self.worker.pipeline = MagicMock(spec=LoRAPipeline)
self.worker.pipeline.stages = [stage]
def _assert_runs_eager(self):
self.assertEqual(self.runner.entries, {})
self.assertEqual(self.runner(**self.kwargs), "eager")
self.assertEqual(self.eager_calls, 1)
def test_set_lora_drops_captured_graphs(self):
self.worker.set_lora("style", "/loras/style.safetensors")
self.worker.pipeline.set_lora.assert_called_once()
self._assert_runs_eager()
def test_merge_and_unmerge_drop_captured_graphs(self):
changes = {
"merge": lambda worker: worker.merge_lora_weights(),
"unmerge": lambda worker: worker.unmerge_lora_weights(),
}
for name, change in changes.items():
with self.subTest(name):
self.setUp()
change(self.worker)
self._assert_runs_eager()
def test_resume_memory_occupation_drops_captured_graphs(self):
self.worker.memory_occupation = MagicMock()
self.worker.resume_memory_occupation()
self._assert_runs_eager()
if __name__ == "__main__":
unittest.main()
@@ -34,6 +34,7 @@ def _make_layer() -> BaseLayerWithLoRA:
def _make_pipeline(layer: BaseLayerWithLoRA) -> _TestLoRAPipeline:
pipeline = object.__new__(_TestLoRAPipeline)
pipeline._stages = []
pipeline.modules = {"transformer": torch.nn.Module()}
pipeline.server_args = SimpleNamespace(
lora_alpha=None,
@@ -3164,6 +3164,110 @@ class TestOffloadDefaults(_CudaPlatformTestCase):
parser.parse_known_args(argv)
class TestBreakableCudaGraphDefault(_CudaPlatformTestCase):
"""Unset BCG turns on only where the served path keeps fixed DiT weights."""
LTX2_TWO_STAGE_SERVE = {
"model_path": "Lightricks/LTX-2",
"num_gpus": 2,
"pipeline_class_name": "LTX2TwoStagePipeline",
"enable_cfg_parallel": True,
"warmup_mode": "server",
}
def _resolve(
self,
*,
memory_gb=141,
available_memory_gb=139,
device_name="NVIDIA H200",
**overrides,
):
with (
patch.object(
current_platform, "device_shares_host_memory", return_value=False
),
patch.object(current_platform, "get_device_name", return_value=device_name),
):
return TestOffloadDefaults._from_dict_with_pipeline_config(
self,
LTX2PipelineConfig(),
memory_gb=memory_gb,
available_memory_gb=available_memory_gb,
kwargs={**self.LTX2_TWO_STAGE_SERVE, **overrides},
)
def test_flagless_ltx2_two_stage_serving_captures_the_model_default(self):
args = self._resolve()
self.assertTrue(args.enable_breakable_cuda_graph)
self.assertEqual(args.warmup_resolutions, ["768x512"])
self.assertEqual(args.warmup_mode, "server")
def test_ltx23_needs_both_stage_dits_resident(self):
resident = self._resolve(model_path="Lightricks/LTX-2.3")
self.assertEqual(resident.ltx2_two_stage_device_mode, "resident")
self.assertTrue(resident.enable_breakable_cuda_graph)
original = self._resolve(
model_path="Lightricks/LTX-2.3", ltx2_two_stage_device_mode="original"
)
self.assertFalse(original.enable_breakable_cuda_graph)
def test_gates_keep_unset_bcg_off(self):
cases = {
"explicit off": {"enable_breakable_cuda_graph": False},
"offline run": {"warmup_mode": None},
"request warmup": {"warmup_mode": "request"},
"memory mode": {"performance_mode": "memory"},
"manual mode": {"performance_mode": "manual"},
"torch.compile": {"enable_torch_compile": True},
"dynamic batching": {"batching_max_size": 2},
"startup LoRA": {"lora_path": "/loras/style.safetensors"},
"ComfyUI": {"comfyui_mode": True},
"one-stage pipeline": {"pipeline_class_name": "LTX2Pipeline"},
"HQ pipeline": {"pipeline_class_name": "LTX2TwoStageHQPipeline"},
"unvalidated checkpoint": {"model_path": "/models/custom-ltx"},
"DiT layerwise offload": {"dit_layerwise_offload": True},
"DiT component offload": {"dit_cpu_offload": True},
"DiT residency selector": {
"component_residency": ["transformer=layerwise-offload"]
},
# below LTX-2's keep-resident threshold auto component-offloads the DiT
"low free memory": {"available_memory_gb": 60},
# the DiT stays resident, but capture outgrows a speed-mode 80 GB card
"80 GB card": {
"memory_gb": 80,
"available_memory_gb": 79,
"device_name": "NVIDIA H100 80GB HBM3",
},
}
for name, overrides in cases.items():
with self.subTest(name):
self.assertFalse(self._resolve(**overrides).enable_breakable_cuda_graph)
def test_platform_gates(self):
args = self._resolve(enable_breakable_cuda_graph=False)
with patch.object(
current_platform, "get_device_total_memory", return_value=141 * 1024**3
):
with patch.object(
current_platform, "device_shares_host_memory", return_value=False
):
self.assertTrue(args._breakable_cuda_graph_default())
# ROCm answers torch.cuda.is_available() but not the platform check
with patch.object(current_platform, "is_cuda", return_value=False):
self.assertFalse(args._breakable_cuda_graph_default())
with patch.object(
current_platform, "device_shares_host_memory", return_value=True
):
self.assertFalse(args._breakable_cuda_graph_default())
def test_explicit_enable_elsewhere_is_unchanged(self):
args = self._resolve(enable_breakable_cuda_graph=True, warmup_mode=None)
self.assertTrue(args.enable_breakable_cuda_graph)
self.assertEqual(args.warmup_mode, "server")
class TestKVGatherDegree(unittest.TestCase):
def test_sp2_defaults_to_kv_gather(self):
args = _from_dict_without_model_resolution(