[diffusion] fix: serve whole graphs after offload-during-compile (#43631)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Mick
2026-10-11 10:03:47 +08:00
committed by GitHub
co-authored by Mick Qian Claude Opus 5.5
parent d7ed5b56f5
commit e4b423256e
2 changed files with 87 additions and 0 deletions
@@ -2068,6 +2068,10 @@ class DenoisingStage(PipelineStage, RolloutDenoisingMixin):
module.disable_offload()
# clear the list for avoid overhead during real request
self._offloaded_dit_modules_for_compile.clear()
# The warmup traced the offload hooks, whose graph breaks inside the
# block loop make Dynamo skip the DiT frame for good. Forget it, so
# this request compiles whole graphs from the cached autotune results.
torch._dynamo.reset()
yield
def forward(
@@ -0,0 +1,83 @@
"""Leaving the offload-during-compile warmup must not keep its graph breaks."""
import unittest
from types import SimpleNamespace
import torch
import torch.nn as nn
from torch._dynamo.testing import CompileCounter
from sglang.multimodal_gen.runtime.pipelines_core.stages.denoising import (
DenoisingStage,
)
from sglang.test.test_utils import CustomTestCase
class _Block(nn.Module):
def forward(self, x):
return x * 2 + 1
class _DiT(nn.Module):
"""Blocks in a loop, each wrapped by an offload hook Dynamo cannot trace."""
def __init__(self):
super().__init__()
self.blocks = nn.ModuleList(_Block() for _ in range(3))
self.hooks = [b.register_forward_pre_hook(_offload_hook) for b in self.blocks]
def disable_offload(self):
for hook in self.hooks:
hook.remove()
def forward(self, x):
for block in self.blocks:
x = block(x)
return x
@torch.compiler.disable
def _offload_hook(module, args):
return None
class TestOffloadDuringCompile(CustomTestCase):
def setUp(self):
torch._dynamo.reset()
self.dit = _DiT()
self.counter = CompileCounter()
self.dit.compile(backend=self.counter)
self.stage = SimpleNamespace(
_offloaded_dit_modules_for_compile=[self.dit],
_move_resident_components_for_warmup=lambda: [],
)
def _forward(self, *, is_warmup):
with DenoisingStage._offload_for_torch_compile_warmup(
self.stage, SimpleNamespace(is_warmup=is_warmup)
):
self.dit(torch.ones(4))
def test_warmup_keeps_the_dit_offloaded(self):
self._forward(is_warmup=True)
self._forward(is_warmup=True)
self.assertEqual(self.stage._offloaded_dit_modules_for_compile, [self.dit])
self.assertTrue(all(b._forward_pre_hooks for b in self.dit.blocks))
def test_serving_compiles_the_whole_dit_after_the_warmup(self):
self._forward(is_warmup=True)
self.counter.frame_count = self.counter.op_count = 0
self._forward(is_warmup=False)
self._forward(is_warmup=False)
# One graph holding all three blocks; without the reset Dynamo keeps
# skipping the frame it gave up on during the warmup and compiles none.
self.assertEqual(self.counter.frame_count, 1)
self.assertEqual(self.counter.op_count, 6)
self.assertEqual(self.stage._offloaded_dit_modules_for_compile, [])
if __name__ == "__main__":
unittest.main()