[diffusion] fix: reset DiT cache state at the start of each request (#42437)

Co-authored-by: Mick <mickjagger19@icloud.com>
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
Wenbo Ji 嵇文博
2026-10-07 23:25:49 +08:00
committed by GitHub
co-authored by Mick Mick Qian
parent 44d10eab5e
commit 00b21d6415
6 changed files with 156 additions and 9 deletions
@@ -637,9 +637,23 @@ class DenoisingStage(PipelineStage, RolloutDenoisingMixin):
)
self._maybe_toggle_quality_fusions(batch)
self._maybe_enable_cache_dit(num_inference_steps, batch)
self._reset_dit_cache_states(batch)
for transformer in filter(None, [self.transformer, self.transformer_2]):
self._maybe_torch_compile(transformer)
def _reset_dit_cache_states(self, batch: Req) -> None:
"""Start every DiT's TeaCache and Spectrum state fresh for this request.
The DiTs reset themselves at denoising step 0, which a boundary expert
(Wan2.2 ``transformer_2``) never sees, so its state would otherwise carry
over from the previous request.
"""
for transformer in filter(None, [self.transformer, self.transformer_2]):
if batch.enable_teacache and hasattr(transformer, "reset_teacache_state"):
transformer.reset_teacache_state()
if batch.enable_spectrum and hasattr(transformer, "reset_spectrum_state"):
transformer.reset_spectrum_state(batch.spectrum_params)
def _maybe_override_attention_backend(
self, batch: Req, *, force_fa_for_self_attention: bool = False
) -> None:
@@ -0,0 +1,140 @@
# SPDX-License-Identifier: Apache-2.0
"""DenoisingStage resets every DiT's cache state at the start of a request.
A boundary expert (Wan2.2 ``transformer_2``) only runs the low-noise steps, so it
never sees denoising step 0, where the DiTs reset TeaCache and Spectrum themselves.
"""
import unittest
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import torch
from sglang.multimodal_gen.configs.sample.spectrum import SpectrumParams
from sglang.multimodal_gen.configs.sample.teacache import TeaCacheParams
from sglang.multimodal_gen.runtime.cache.spectrum import SpectrumMixin
from sglang.multimodal_gen.runtime.cache.teacache import TeaCacheMixin
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
from sglang.multimodal_gen.runtime.models.dits.wanvideo import WanTransformer3DModel
from sglang.multimodal_gen.runtime.pipelines_core.stages.denoising import DenoisingStage
from sglang.test.test_utils import CustomTestCase
STEPS = 8
FIRST_LOW_NOISE_STEP = 4 # where the second expert takes over
class _Expert(SpectrumMixin, TeaCacheMixin):
def __init__(self):
self.prefix = "wan"
self._init_spectrum_state()
self._init_teacache_state()
class TestDenoisingCacheReset(CustomTestCase):
def setUp(self):
server_args = patch(
"sglang.multimodal_gen.runtime.server_args.get_global_server_args",
return_value=SimpleNamespace(enable_cfg_parallel=False),
)
server_args.start()
self.addCleanup(server_args.stop)
def _batch(self, spectrum=False, teacache=False):
return SimpleNamespace(
enable_spectrum=spectrum,
spectrum_params=SpectrumParams(
warmup_steps=1, window_size=2.0, flex_window=0.5, m=1
),
enable_teacache=teacache,
teacache_params=TeaCacheParams(
teacache_thresh=0.5, coefficients=[1.0, 0.0], start_skipping=1
),
do_classifier_free_guidance=False,
is_cfg_negative=False,
num_inference_steps=STEPS,
debug=False,
)
def _stage(self, expert, batch):
stage = DenoisingStage.__new__(DenoisingStage)
stage.transformer, stage.transformer_2 = MagicMock(spec=[]), expert
stage._reset_dit_cache_states(batch)
def _spectrum_request(self, expert, batch, shape):
"""Run the second expert's steps; return (ran blocks, output) per step."""
outputs = []
for step in range(FIRST_LOW_NOISE_STEP, STEPS):
features = torch.full(shape, float(step))
with set_forward_context(step, None, forward_batch=batch):
if expert.begin_spectrum_step():
expert.spectrum_record_features(features)
outputs.append((True, features))
else:
outputs.append(
(
False,
expert.spectrum_predict_features(
torch.zeros_like(features)
),
)
)
return outputs
def _teacache_request(self, expert, batch, offset):
decisions = []
for step in range(FIRST_LOW_NOISE_STEP, STEPS):
inputs = torch.full((2,), offset + step * 0.01)
with set_forward_context(step, None, batch):
skip = WanTransformer3DModel.should_skip_forward_for_cached_states(
expert, timestep_proj=inputs, temb=inputs
)
expert.cnt += 1
decisions.append(skip)
return decisions
def test_second_expert_spectrum_restarts_like_fresh(self):
"""A changed feature shape must not hit the previous request's forecaster."""
batch = self._batch(spectrum=True)
reused, fresh = _Expert(), _Expert()
self._spectrum_request(reused, batch, (2, 3))
self._stage(reused, batch)
self._stage(fresh, batch)
for (ran, out), (fresh_ran, fresh_out) in zip(
self._spectrum_request(reused, batch, (3, 4)),
self._spectrum_request(fresh, batch, (3, 4)),
strict=True,
):
self.assertEqual(ran, fresh_ran)
torch.testing.assert_close(out, fresh_out)
def test_second_expert_teacache_restarts_like_fresh(self):
"""Counters, residuals and L1 history must not carry over."""
batch = self._batch(teacache=True)
reused, fresh = _Expert(), _Expert()
self._teacache_request(reused, batch, offset=1.0)
reused.previous_residual = torch.ones(2)
self._stage(reused, batch)
self.assertEqual(reused.cnt, 0)
self.assertIsNone(reused.previous_modulated_input)
self.assertIsNone(reused.previous_residual)
self.assertEqual(reused.accumulated_rel_l1_distance, 0.0)
self.assertEqual(
self._teacache_request(reused, batch, offset=5.0),
self._teacache_request(fresh, batch, offset=5.0),
)
def test_disabled_caches_and_plain_modules_are_left_alone(self):
"""Only enabled caches reset, and modules without cache state are skipped."""
expert = MagicMock()
self._stage(expert, self._batch())
expert.reset_teacache_state.assert_not_called()
expert.reset_spectrum_state.assert_not_called()
batch = self._batch(spectrum=True, teacache=True)
self._stage(expert, batch)
expert.reset_teacache_state.assert_called_once_with()
expert.reset_spectrum_state.assert_called_once_with(batch.spectrum_params)
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,87 @@
"""Unit tests for FLUX.2 ModelOpt FP8 norm+quant activation gates."""
import unittest
from types import SimpleNamespace
import torch
import torch.nn as nn
from sglang.multimodal_gen.runtime.layers.quantization.modelopt_quant import (
ModelOptFp8LinearMethod,
)
from sglang.multimodal_gen.runtime.models.dits.flux_2 import (
Flux2SingleTransformerBlock,
Flux2TransformerBlock,
)
from sglang.test.test_utils import CustomTestCase
def _fp8_linear(input_scale: float) -> nn.Module:
linear = nn.Module()
linear.quant_method = object.__new__(ModelOptFp8LinearMethod)
linear.register_parameter(
"input_scale",
nn.Parameter(
torch.tensor(input_scale, dtype=torch.float32, device="cuda"),
requires_grad=False,
),
)
return linear
def _double_block(scales: tuple[float, float, float]) -> Flux2TransformerBlock:
block = object.__new__(Flux2TransformerBlock)
nn.Module.__init__(block)
block.attn = SimpleNamespace(
use_fused_qkv=False,
use_fused_added_qkv=False,
to_q=_fp8_linear(scales[0]),
to_k=_fp8_linear(scales[1]),
to_v=_fp8_linear(scales[2]),
add_q_proj=_fp8_linear(scales[0]),
add_k_proj=_fp8_linear(scales[1]),
add_v_proj=_fp8_linear(scales[2]),
)
block.ff = SimpleNamespace(linear_in=_fp8_linear(scales[0]))
block.ff_context = SimpleNamespace(linear_in=_fp8_linear(scales[0]))
block._fp8_img_attn_norm_quant = False
block._fp8_txt_attn_norm_quant = False
block._fp8_img_ff_norm_quant = False
block._fp8_txt_ff_norm_quant = False
return block
class TestFlux2Fp8NormQuantGate(CustomTestCase):
def test_qkv_requires_identical_input_scales(self) -> None:
matching = _double_block((0.25, 0.25, 0.25))
mismatched = _double_block((0.25, 0.5, 0.25))
matching.configure_fp8_norm_quant()
mismatched.configure_fp8_norm_quant()
self.assertTrue(matching._fp8_img_attn_norm_quant)
self.assertTrue(matching._fp8_txt_attn_norm_quant)
self.assertFalse(mismatched._fp8_img_attn_norm_quant)
self.assertFalse(mismatched._fp8_txt_attn_norm_quant)
def test_single_block_uses_merged_projection_scale(self) -> None:
block = object.__new__(Flux2SingleTransformerBlock)
nn.Module.__init__(block)
block.attn = SimpleNamespace(to_qkv_mlp_proj=_fp8_linear(0.25))
block._fp8_norm_quant = False
block.configure_fp8_norm_quant()
self.assertTrue(block._fp8_norm_quant)
def test_nonpositive_scale_keeps_fusion_disabled(self) -> None:
block = _double_block((0.0, 0.0, 0.0))
block.configure_fp8_norm_quant()
self.assertFalse(block._fp8_img_attn_norm_quant)
self.assertFalse(block._fp8_img_ff_norm_quant)
if __name__ == "__main__":
unittest.main()
@@ -279,6 +279,8 @@ def test_distilled_denoising_calls_the_dit_once_per_step_without_guidance(
scheduler=scheduler,
guidance_scale=1.0,
do_classifier_free_guidance=False,
enable_teacache=False,
enable_spectrum=False,
extra={},
prompt_embeds=[torch.randn(1, text_len, 8)],
negative_prompt_embeds=[torch.randn(1, text_len + 2, 8)],
@@ -0,0 +1,168 @@
# SPDX-License-Identifier: Apache-2.0
import json
import tempfile
import unittest
from functools import partial
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch
import torch
from safetensors.torch import load_file, save_file
from sglang.multimodal_gen.configs.models.dits.minimax_h3 import MiniMaxH3DiTArchConfig
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
from sglang.multimodal_gen.runtime.models.dits.minimax_h3 import MiniMaxH3FinalLayer
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.constants import (
MINIMAX_H3_SIGMAS_EXTRA_KEY,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.stages.timestep_preparation import (
MiniMaxH3TimestepPreparationStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.time_request import (
minimax_h3_time_shift_sigmas,
)
from sglang.multimodal_gen.tools import build_minimax_h3_pdd_weights as build
from sglang.test.test_utils import CustomTestCase
def _make_heads() -> dict[str, torch.Tensor]:
heads = {}
for name, width in (("video_out", 6), ("audio_out", 4)):
heads[f"{name}.weight"] = (
torch.arange(8 * width * 5, dtype=torch.float32).reshape(8, width, 5) / 100
)
heads[f"{name}.bias"] = (
torch.arange(8 * width, dtype=torch.float32).reshape(8, width) / 100
)
return heads
class TestMiniMaxH3PDDOffline(CustomTestCase):
def test_tp_projection_matches_full_head(self):
with tempfile.TemporaryDirectory() as temp:
root = Path(temp)
heads = _make_heads()
path = root / "heads.safetensors"
save_file(heads, str(path))
arch = MiniMaxH3DiTArchConfig(
hidden_size=5, latents_dim=6, audio_latents_dim=4, patch_size=(1, 1, 1)
)
layers = []
for rank in range(2):
with patch(
"sglang.multimodal_gen.runtime.layers.linear.get_tp_group",
return_value=SimpleNamespace(world_size=2, rank_in_group=rank),
):
layer = MiniMaxH3FinalLayer(
arch, None, prefix="final_layer", use_adaln_cache=True
)
layer.load_pdd_fused_heads(str(path))
layers.append(layer)
h = torch.arange(15, dtype=torch.float32).reshape(3, 5)
for step in (0, 7):
with self.subTest(step=step), set_forward_context(step, None):
outputs = [layer._project(h) for layer in layers]
for i, name in enumerate(("video_out", "audio_out")):
expected = torch.nn.functional.linear(
h,
heads[f"{name}.weight"][step],
heads[f"{name}.bias"][step],
)
torch.testing.assert_close(
torch.cat([output[i] for output in outputs], dim=-1),
expected,
)
def test_schedule_validation_and_warmup(self):
apply_schedule = partial(
MiniMaxH3TimestepPreparationStage._apply_pdd_schedule,
SimpleNamespace(
_pdd_config={
"num_inference_steps": 9,
"video_shift": 12.0,
"audio_shift": 3.0,
}
),
)
sigmas = {
name: minimax_h3_time_shift_sigmas(num_steps=9, shift_scale=shift)
for name, shift in (("video", 12.0), ("audio", 3.0))
}
batch = SimpleNamespace(
is_warmup=False, extra={MINIMAX_H3_SIGMAS_EXTRA_KEY: sigmas}
)
apply_schedule(batch)
wrong_shift = minimax_h3_time_shift_sigmas(num_steps=9, shift_scale=6.0)
for name, invalid in (
("video", sigmas["video"][:-1]),
("video", sigmas["video"] + [0.0]),
("video", wrong_shift),
("audio", wrong_shift),
):
batch.extra[MINIMAX_H3_SIGMAS_EXTRA_KEY] = {**sigmas, name: invalid}
with (
self.subTest(modality=name, sigmas=invalid),
self.assertRaisesRegex(ValueError, "match the fused heads"),
):
apply_schedule(batch)
batch.is_warmup = True
batch.num_inference_steps = 3
apply_schedule(batch)
self.assertEqual(
batch.extra[MINIMAX_H3_SIGMAS_EXTRA_KEY],
{name: values[:3] for name, values in sigmas.items()},
)
def test_conversion_keeps_heads_outside_transformer(self):
with tempfile.TemporaryDirectory() as temp:
root = Path(temp)
heads = _make_heads()
base = root / "base"
base.mkdir()
weight_name = "blocks.0.attn.out_proj.weight"
save_file({weight_name: torch.zeros(2, 3)}, str(base / "model.safetensors"))
(base / "config.json").write_text("{}")
index = {"weight_map": {weight_name: "model.safetensors"}}
(base / "model.safetensors.index.json").write_text(json.dumps(index))
lora = {
"transformer_blocks.0.attn.to_out.0.lora_down": torch.ones(1, 3),
"transformer_blocks.0.attn.to_out.0.lora_up": torch.ones(2, 1),
}
for key, value in heads.items():
lora[
key.replace("video_out", "proj_out").replace(
"audio_out", "audio_proj_out"
)
] = value
lora_path = root / "lora.safetensors"
save_file(
lora,
str(lora_path),
metadata={
"lora_rank": "1",
"pdd_num_steps": "8",
"pdd_block_size": "2",
},
)
out = root / "out"
build.main([str(base), str(lora_path), str(out)])
transformer = out / "transformer"
self.assertEqual(
list(transformer.glob("*.safetensors")),
[transformer / "model.safetensors"],
)
torch.testing.assert_close(
load_file(str(transformer / "model.safetensors"))[weight_name],
torch.ones(2, 3),
)
self.assertEqual(
json.loads((transformer / "model.safetensors.index.json").read_text()),
index,
)
if __name__ == "__main__":
unittest.main(verbosity=3)
@@ -0,0 +1,98 @@
"""Unit tests for Qwen-Image ModelOpt FP8 norm+quant activation gates."""
import unittest
from types import SimpleNamespace
from unittest.mock import patch
import torch
import torch.nn as nn
from sglang.multimodal_gen.runtime.layers.quantization.modelopt_quant import (
ModelOptFp8LinearMethod,
)
from sglang.multimodal_gen.runtime.models.dits.qwen_image import (
QwenImageTransformerBlock,
)
from sglang.test.test_utils import CustomTestCase
def _fp8_linear(input_scale: float) -> nn.Module:
linear = nn.Module()
linear.quant_method = object.__new__(ModelOptFp8LinearMethod)
linear.register_parameter(
"input_scale",
nn.Parameter(
torch.tensor(input_scale, dtype=torch.float32, device="cuda"),
requires_grad=False,
),
)
return linear
def _attention(*, fused: bool, scales: tuple[float, ...]) -> SimpleNamespace:
if fused:
return SimpleNamespace(
use_fused_qkv=True,
to_qkv=_fp8_linear(scales[0]),
added_kv_proj_dim=None,
)
return SimpleNamespace(
use_fused_qkv=False,
to_q=_fp8_linear(scales[0]),
to_k=_fp8_linear(scales[1]),
to_v=_fp8_linear(scales[2]),
added_kv_proj_dim=None,
)
def _block(attn: SimpleNamespace) -> QwenImageTransformerBlock:
block = object.__new__(QwenImageTransformerBlock)
nn.Module.__init__(block)
block.dim = 3072
block.zero_cond_t = False
block.attn = attn
block.img_mlp = None
block.txt_mlp = None
block._fp8_img_attn_norm_quant = False
block._fp8_txt_attn_norm_quant = False
block._fp8_img_mlp_norm_quant = False
block._fp8_txt_mlp_norm_quant = False
return block
@patch("torch.cuda.get_device_capability", return_value=(10, 0))
@patch("torch.cuda.is_available", return_value=True)
class TestQwenImageFp8NormQuantGate(CustomTestCase):
def test_separate_qkv_requires_identical_input_scales(
self, _is_available, _capability
) -> None:
matching = _block(_attention(fused=False, scales=(0.25, 0.25, 0.25)))
mismatched = _block(_attention(fused=False, scales=(0.25, 0.5, 0.25)))
matching.configure_fp8_norm_quant()
mismatched.configure_fp8_norm_quant()
self.assertTrue(matching._fp8_img_attn_norm_quant)
self.assertFalse(mismatched._fp8_img_attn_norm_quant)
def test_merged_qkv_uses_its_materialized_input_scale(
self, _is_available, _capability
) -> None:
block = _block(_attention(fused=True, scales=(0.25,)))
block.configure_fp8_norm_quant()
self.assertTrue(block._fp8_img_attn_norm_quant)
def test_nonpositive_scale_keeps_fusion_disabled(
self, _is_available, _capability
) -> None:
block = _block(_attention(fused=True, scales=(0.0,)))
block.configure_fp8_norm_quant()
self.assertFalse(block._fp8_img_attn_norm_quant)
if __name__ == "__main__":
unittest.main()