[diffusion] fix: reset DiT cache state at the start of each request (#42437)
Co-authored-by: Mick <mickjagger19@icloud.com> Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
co-authored by
Mick
Mick Qian
parent
44d10eab5e
commit
00b21d6415
@@ -637,9 +637,23 @@ class DenoisingStage(PipelineStage, RolloutDenoisingMixin):
|
||||
)
|
||||
self._maybe_toggle_quality_fusions(batch)
|
||||
self._maybe_enable_cache_dit(num_inference_steps, batch)
|
||||
self._reset_dit_cache_states(batch)
|
||||
for transformer in filter(None, [self.transformer, self.transformer_2]):
|
||||
self._maybe_torch_compile(transformer)
|
||||
|
||||
def _reset_dit_cache_states(self, batch: Req) -> None:
|
||||
"""Start every DiT's TeaCache and Spectrum state fresh for this request.
|
||||
|
||||
The DiTs reset themselves at denoising step 0, which a boundary expert
|
||||
(Wan2.2 ``transformer_2``) never sees, so its state would otherwise carry
|
||||
over from the previous request.
|
||||
"""
|
||||
for transformer in filter(None, [self.transformer, self.transformer_2]):
|
||||
if batch.enable_teacache and hasattr(transformer, "reset_teacache_state"):
|
||||
transformer.reset_teacache_state()
|
||||
if batch.enable_spectrum and hasattr(transformer, "reset_spectrum_state"):
|
||||
transformer.reset_spectrum_state(batch.spectrum_params)
|
||||
|
||||
def _maybe_override_attention_backend(
|
||||
self, batch: Req, *, force_fa_for_self_attention: bool = False
|
||||
) -> None:
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""DenoisingStage resets every DiT's cache state at the start of a request.
|
||||
|
||||
A boundary expert (Wan2.2 ``transformer_2``) only runs the low-noise steps, so it
|
||||
never sees denoising step 0, where the DiTs reset TeaCache and Spectrum themselves.
|
||||
"""
|
||||
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.configs.sample.spectrum import SpectrumParams
|
||||
from sglang.multimodal_gen.configs.sample.teacache import TeaCacheParams
|
||||
from sglang.multimodal_gen.runtime.cache.spectrum import SpectrumMixin
|
||||
from sglang.multimodal_gen.runtime.cache.teacache import TeaCacheMixin
|
||||
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
|
||||
from sglang.multimodal_gen.runtime.models.dits.wanvideo import WanTransformer3DModel
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.denoising import DenoisingStage
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
STEPS = 8
|
||||
FIRST_LOW_NOISE_STEP = 4 # where the second expert takes over
|
||||
|
||||
|
||||
class _Expert(SpectrumMixin, TeaCacheMixin):
|
||||
def __init__(self):
|
||||
self.prefix = "wan"
|
||||
self._init_spectrum_state()
|
||||
self._init_teacache_state()
|
||||
|
||||
|
||||
class TestDenoisingCacheReset(CustomTestCase):
|
||||
def setUp(self):
|
||||
server_args = patch(
|
||||
"sglang.multimodal_gen.runtime.server_args.get_global_server_args",
|
||||
return_value=SimpleNamespace(enable_cfg_parallel=False),
|
||||
)
|
||||
server_args.start()
|
||||
self.addCleanup(server_args.stop)
|
||||
|
||||
def _batch(self, spectrum=False, teacache=False):
|
||||
return SimpleNamespace(
|
||||
enable_spectrum=spectrum,
|
||||
spectrum_params=SpectrumParams(
|
||||
warmup_steps=1, window_size=2.0, flex_window=0.5, m=1
|
||||
),
|
||||
enable_teacache=teacache,
|
||||
teacache_params=TeaCacheParams(
|
||||
teacache_thresh=0.5, coefficients=[1.0, 0.0], start_skipping=1
|
||||
),
|
||||
do_classifier_free_guidance=False,
|
||||
is_cfg_negative=False,
|
||||
num_inference_steps=STEPS,
|
||||
debug=False,
|
||||
)
|
||||
|
||||
def _stage(self, expert, batch):
|
||||
stage = DenoisingStage.__new__(DenoisingStage)
|
||||
stage.transformer, stage.transformer_2 = MagicMock(spec=[]), expert
|
||||
stage._reset_dit_cache_states(batch)
|
||||
|
||||
def _spectrum_request(self, expert, batch, shape):
|
||||
"""Run the second expert's steps; return (ran blocks, output) per step."""
|
||||
outputs = []
|
||||
for step in range(FIRST_LOW_NOISE_STEP, STEPS):
|
||||
features = torch.full(shape, float(step))
|
||||
with set_forward_context(step, None, forward_batch=batch):
|
||||
if expert.begin_spectrum_step():
|
||||
expert.spectrum_record_features(features)
|
||||
outputs.append((True, features))
|
||||
else:
|
||||
outputs.append(
|
||||
(
|
||||
False,
|
||||
expert.spectrum_predict_features(
|
||||
torch.zeros_like(features)
|
||||
),
|
||||
)
|
||||
)
|
||||
return outputs
|
||||
|
||||
def _teacache_request(self, expert, batch, offset):
|
||||
decisions = []
|
||||
for step in range(FIRST_LOW_NOISE_STEP, STEPS):
|
||||
inputs = torch.full((2,), offset + step * 0.01)
|
||||
with set_forward_context(step, None, batch):
|
||||
skip = WanTransformer3DModel.should_skip_forward_for_cached_states(
|
||||
expert, timestep_proj=inputs, temb=inputs
|
||||
)
|
||||
expert.cnt += 1
|
||||
decisions.append(skip)
|
||||
return decisions
|
||||
|
||||
def test_second_expert_spectrum_restarts_like_fresh(self):
|
||||
"""A changed feature shape must not hit the previous request's forecaster."""
|
||||
batch = self._batch(spectrum=True)
|
||||
reused, fresh = _Expert(), _Expert()
|
||||
self._spectrum_request(reused, batch, (2, 3))
|
||||
self._stage(reused, batch)
|
||||
self._stage(fresh, batch)
|
||||
for (ran, out), (fresh_ran, fresh_out) in zip(
|
||||
self._spectrum_request(reused, batch, (3, 4)),
|
||||
self._spectrum_request(fresh, batch, (3, 4)),
|
||||
strict=True,
|
||||
):
|
||||
self.assertEqual(ran, fresh_ran)
|
||||
torch.testing.assert_close(out, fresh_out)
|
||||
|
||||
def test_second_expert_teacache_restarts_like_fresh(self):
|
||||
"""Counters, residuals and L1 history must not carry over."""
|
||||
batch = self._batch(teacache=True)
|
||||
reused, fresh = _Expert(), _Expert()
|
||||
self._teacache_request(reused, batch, offset=1.0)
|
||||
reused.previous_residual = torch.ones(2)
|
||||
self._stage(reused, batch)
|
||||
self.assertEqual(reused.cnt, 0)
|
||||
self.assertIsNone(reused.previous_modulated_input)
|
||||
self.assertIsNone(reused.previous_residual)
|
||||
self.assertEqual(reused.accumulated_rel_l1_distance, 0.0)
|
||||
self.assertEqual(
|
||||
self._teacache_request(reused, batch, offset=5.0),
|
||||
self._teacache_request(fresh, batch, offset=5.0),
|
||||
)
|
||||
|
||||
def test_disabled_caches_and_plain_modules_are_left_alone(self):
|
||||
"""Only enabled caches reset, and modules without cache state are skipped."""
|
||||
expert = MagicMock()
|
||||
self._stage(expert, self._batch())
|
||||
expert.reset_teacache_state.assert_not_called()
|
||||
expert.reset_spectrum_state.assert_not_called()
|
||||
batch = self._batch(spectrum=True, teacache=True)
|
||||
self._stage(expert, batch)
|
||||
expert.reset_teacache_state.assert_called_once_with()
|
||||
expert.reset_spectrum_state.assert_called_once_with(batch.spectrum_params)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,87 @@
|
||||
"""Unit tests for FLUX.2 ModelOpt FP8 norm+quant activation gates."""
|
||||
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.modelopt_quant import (
|
||||
ModelOptFp8LinearMethod,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.dits.flux_2 import (
|
||||
Flux2SingleTransformerBlock,
|
||||
Flux2TransformerBlock,
|
||||
)
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
|
||||
def _fp8_linear(input_scale: float) -> nn.Module:
|
||||
linear = nn.Module()
|
||||
linear.quant_method = object.__new__(ModelOptFp8LinearMethod)
|
||||
linear.register_parameter(
|
||||
"input_scale",
|
||||
nn.Parameter(
|
||||
torch.tensor(input_scale, dtype=torch.float32, device="cuda"),
|
||||
requires_grad=False,
|
||||
),
|
||||
)
|
||||
return linear
|
||||
|
||||
|
||||
def _double_block(scales: tuple[float, float, float]) -> Flux2TransformerBlock:
|
||||
block = object.__new__(Flux2TransformerBlock)
|
||||
nn.Module.__init__(block)
|
||||
block.attn = SimpleNamespace(
|
||||
use_fused_qkv=False,
|
||||
use_fused_added_qkv=False,
|
||||
to_q=_fp8_linear(scales[0]),
|
||||
to_k=_fp8_linear(scales[1]),
|
||||
to_v=_fp8_linear(scales[2]),
|
||||
add_q_proj=_fp8_linear(scales[0]),
|
||||
add_k_proj=_fp8_linear(scales[1]),
|
||||
add_v_proj=_fp8_linear(scales[2]),
|
||||
)
|
||||
block.ff = SimpleNamespace(linear_in=_fp8_linear(scales[0]))
|
||||
block.ff_context = SimpleNamespace(linear_in=_fp8_linear(scales[0]))
|
||||
block._fp8_img_attn_norm_quant = False
|
||||
block._fp8_txt_attn_norm_quant = False
|
||||
block._fp8_img_ff_norm_quant = False
|
||||
block._fp8_txt_ff_norm_quant = False
|
||||
return block
|
||||
|
||||
|
||||
class TestFlux2Fp8NormQuantGate(CustomTestCase):
|
||||
def test_qkv_requires_identical_input_scales(self) -> None:
|
||||
matching = _double_block((0.25, 0.25, 0.25))
|
||||
mismatched = _double_block((0.25, 0.5, 0.25))
|
||||
|
||||
matching.configure_fp8_norm_quant()
|
||||
mismatched.configure_fp8_norm_quant()
|
||||
|
||||
self.assertTrue(matching._fp8_img_attn_norm_quant)
|
||||
self.assertTrue(matching._fp8_txt_attn_norm_quant)
|
||||
self.assertFalse(mismatched._fp8_img_attn_norm_quant)
|
||||
self.assertFalse(mismatched._fp8_txt_attn_norm_quant)
|
||||
|
||||
def test_single_block_uses_merged_projection_scale(self) -> None:
|
||||
block = object.__new__(Flux2SingleTransformerBlock)
|
||||
nn.Module.__init__(block)
|
||||
block.attn = SimpleNamespace(to_qkv_mlp_proj=_fp8_linear(0.25))
|
||||
block._fp8_norm_quant = False
|
||||
|
||||
block.configure_fp8_norm_quant()
|
||||
|
||||
self.assertTrue(block._fp8_norm_quant)
|
||||
|
||||
def test_nonpositive_scale_keeps_fusion_disabled(self) -> None:
|
||||
block = _double_block((0.0, 0.0, 0.0))
|
||||
|
||||
block.configure_fp8_norm_quant()
|
||||
|
||||
self.assertFalse(block._fp8_img_attn_norm_quant)
|
||||
self.assertFalse(block._fp8_img_ff_norm_quant)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -279,6 +279,8 @@ def test_distilled_denoising_calls_the_dit_once_per_step_without_guidance(
|
||||
scheduler=scheduler,
|
||||
guidance_scale=1.0,
|
||||
do_classifier_free_guidance=False,
|
||||
enable_teacache=False,
|
||||
enable_spectrum=False,
|
||||
extra={},
|
||||
prompt_embeds=[torch.randn(1, text_len, 8)],
|
||||
negative_prompt_embeds=[torch.randn(1, text_len + 2, 8)],
|
||||
|
||||
@@ -0,0 +1,168 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import torch
|
||||
from safetensors.torch import load_file, save_file
|
||||
|
||||
from sglang.multimodal_gen.configs.models.dits.minimax_h3 import MiniMaxH3DiTArchConfig
|
||||
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
|
||||
from sglang.multimodal_gen.runtime.models.dits.minimax_h3 import MiniMaxH3FinalLayer
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.constants import (
|
||||
MINIMAX_H3_SIGMAS_EXTRA_KEY,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.stages.timestep_preparation import (
|
||||
MiniMaxH3TimestepPreparationStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.time_request import (
|
||||
minimax_h3_time_shift_sigmas,
|
||||
)
|
||||
from sglang.multimodal_gen.tools import build_minimax_h3_pdd_weights as build
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
|
||||
def _make_heads() -> dict[str, torch.Tensor]:
|
||||
heads = {}
|
||||
for name, width in (("video_out", 6), ("audio_out", 4)):
|
||||
heads[f"{name}.weight"] = (
|
||||
torch.arange(8 * width * 5, dtype=torch.float32).reshape(8, width, 5) / 100
|
||||
)
|
||||
heads[f"{name}.bias"] = (
|
||||
torch.arange(8 * width, dtype=torch.float32).reshape(8, width) / 100
|
||||
)
|
||||
return heads
|
||||
|
||||
|
||||
class TestMiniMaxH3PDDOffline(CustomTestCase):
|
||||
def test_tp_projection_matches_full_head(self):
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
root = Path(temp)
|
||||
heads = _make_heads()
|
||||
path = root / "heads.safetensors"
|
||||
save_file(heads, str(path))
|
||||
arch = MiniMaxH3DiTArchConfig(
|
||||
hidden_size=5, latents_dim=6, audio_latents_dim=4, patch_size=(1, 1, 1)
|
||||
)
|
||||
layers = []
|
||||
for rank in range(2):
|
||||
with patch(
|
||||
"sglang.multimodal_gen.runtime.layers.linear.get_tp_group",
|
||||
return_value=SimpleNamespace(world_size=2, rank_in_group=rank),
|
||||
):
|
||||
layer = MiniMaxH3FinalLayer(
|
||||
arch, None, prefix="final_layer", use_adaln_cache=True
|
||||
)
|
||||
layer.load_pdd_fused_heads(str(path))
|
||||
layers.append(layer)
|
||||
|
||||
h = torch.arange(15, dtype=torch.float32).reshape(3, 5)
|
||||
for step in (0, 7):
|
||||
with self.subTest(step=step), set_forward_context(step, None):
|
||||
outputs = [layer._project(h) for layer in layers]
|
||||
for i, name in enumerate(("video_out", "audio_out")):
|
||||
expected = torch.nn.functional.linear(
|
||||
h,
|
||||
heads[f"{name}.weight"][step],
|
||||
heads[f"{name}.bias"][step],
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
torch.cat([output[i] for output in outputs], dim=-1),
|
||||
expected,
|
||||
)
|
||||
|
||||
def test_schedule_validation_and_warmup(self):
|
||||
apply_schedule = partial(
|
||||
MiniMaxH3TimestepPreparationStage._apply_pdd_schedule,
|
||||
SimpleNamespace(
|
||||
_pdd_config={
|
||||
"num_inference_steps": 9,
|
||||
"video_shift": 12.0,
|
||||
"audio_shift": 3.0,
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
sigmas = {
|
||||
name: minimax_h3_time_shift_sigmas(num_steps=9, shift_scale=shift)
|
||||
for name, shift in (("video", 12.0), ("audio", 3.0))
|
||||
}
|
||||
batch = SimpleNamespace(
|
||||
is_warmup=False, extra={MINIMAX_H3_SIGMAS_EXTRA_KEY: sigmas}
|
||||
)
|
||||
apply_schedule(batch)
|
||||
wrong_shift = minimax_h3_time_shift_sigmas(num_steps=9, shift_scale=6.0)
|
||||
for name, invalid in (
|
||||
("video", sigmas["video"][:-1]),
|
||||
("video", sigmas["video"] + [0.0]),
|
||||
("video", wrong_shift),
|
||||
("audio", wrong_shift),
|
||||
):
|
||||
batch.extra[MINIMAX_H3_SIGMAS_EXTRA_KEY] = {**sigmas, name: invalid}
|
||||
with (
|
||||
self.subTest(modality=name, sigmas=invalid),
|
||||
self.assertRaisesRegex(ValueError, "match the fused heads"),
|
||||
):
|
||||
apply_schedule(batch)
|
||||
batch.is_warmup = True
|
||||
batch.num_inference_steps = 3
|
||||
apply_schedule(batch)
|
||||
self.assertEqual(
|
||||
batch.extra[MINIMAX_H3_SIGMAS_EXTRA_KEY],
|
||||
{name: values[:3] for name, values in sigmas.items()},
|
||||
)
|
||||
|
||||
def test_conversion_keeps_heads_outside_transformer(self):
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
root = Path(temp)
|
||||
heads = _make_heads()
|
||||
base = root / "base"
|
||||
base.mkdir()
|
||||
weight_name = "blocks.0.attn.out_proj.weight"
|
||||
save_file({weight_name: torch.zeros(2, 3)}, str(base / "model.safetensors"))
|
||||
(base / "config.json").write_text("{}")
|
||||
index = {"weight_map": {weight_name: "model.safetensors"}}
|
||||
(base / "model.safetensors.index.json").write_text(json.dumps(index))
|
||||
lora = {
|
||||
"transformer_blocks.0.attn.to_out.0.lora_down": torch.ones(1, 3),
|
||||
"transformer_blocks.0.attn.to_out.0.lora_up": torch.ones(2, 1),
|
||||
}
|
||||
for key, value in heads.items():
|
||||
lora[
|
||||
key.replace("video_out", "proj_out").replace(
|
||||
"audio_out", "audio_proj_out"
|
||||
)
|
||||
] = value
|
||||
lora_path = root / "lora.safetensors"
|
||||
save_file(
|
||||
lora,
|
||||
str(lora_path),
|
||||
metadata={
|
||||
"lora_rank": "1",
|
||||
"pdd_num_steps": "8",
|
||||
"pdd_block_size": "2",
|
||||
},
|
||||
)
|
||||
out = root / "out"
|
||||
build.main([str(base), str(lora_path), str(out)])
|
||||
transformer = out / "transformer"
|
||||
self.assertEqual(
|
||||
list(transformer.glob("*.safetensors")),
|
||||
[transformer / "model.safetensors"],
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
load_file(str(transformer / "model.safetensors"))[weight_name],
|
||||
torch.ones(2, 3),
|
||||
)
|
||||
self.assertEqual(
|
||||
json.loads((transformer / "model.safetensors.index.json").read_text()),
|
||||
index,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=3)
|
||||
@@ -0,0 +1,98 @@
|
||||
"""Unit tests for Qwen-Image ModelOpt FP8 norm+quant activation gates."""
|
||||
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.modelopt_quant import (
|
||||
ModelOptFp8LinearMethod,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.dits.qwen_image import (
|
||||
QwenImageTransformerBlock,
|
||||
)
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
|
||||
def _fp8_linear(input_scale: float) -> nn.Module:
|
||||
linear = nn.Module()
|
||||
linear.quant_method = object.__new__(ModelOptFp8LinearMethod)
|
||||
linear.register_parameter(
|
||||
"input_scale",
|
||||
nn.Parameter(
|
||||
torch.tensor(input_scale, dtype=torch.float32, device="cuda"),
|
||||
requires_grad=False,
|
||||
),
|
||||
)
|
||||
return linear
|
||||
|
||||
|
||||
def _attention(*, fused: bool, scales: tuple[float, ...]) -> SimpleNamespace:
|
||||
if fused:
|
||||
return SimpleNamespace(
|
||||
use_fused_qkv=True,
|
||||
to_qkv=_fp8_linear(scales[0]),
|
||||
added_kv_proj_dim=None,
|
||||
)
|
||||
return SimpleNamespace(
|
||||
use_fused_qkv=False,
|
||||
to_q=_fp8_linear(scales[0]),
|
||||
to_k=_fp8_linear(scales[1]),
|
||||
to_v=_fp8_linear(scales[2]),
|
||||
added_kv_proj_dim=None,
|
||||
)
|
||||
|
||||
|
||||
def _block(attn: SimpleNamespace) -> QwenImageTransformerBlock:
|
||||
block = object.__new__(QwenImageTransformerBlock)
|
||||
nn.Module.__init__(block)
|
||||
block.dim = 3072
|
||||
block.zero_cond_t = False
|
||||
block.attn = attn
|
||||
block.img_mlp = None
|
||||
block.txt_mlp = None
|
||||
block._fp8_img_attn_norm_quant = False
|
||||
block._fp8_txt_attn_norm_quant = False
|
||||
block._fp8_img_mlp_norm_quant = False
|
||||
block._fp8_txt_mlp_norm_quant = False
|
||||
return block
|
||||
|
||||
|
||||
@patch("torch.cuda.get_device_capability", return_value=(10, 0))
|
||||
@patch("torch.cuda.is_available", return_value=True)
|
||||
class TestQwenImageFp8NormQuantGate(CustomTestCase):
|
||||
def test_separate_qkv_requires_identical_input_scales(
|
||||
self, _is_available, _capability
|
||||
) -> None:
|
||||
matching = _block(_attention(fused=False, scales=(0.25, 0.25, 0.25)))
|
||||
mismatched = _block(_attention(fused=False, scales=(0.25, 0.5, 0.25)))
|
||||
|
||||
matching.configure_fp8_norm_quant()
|
||||
mismatched.configure_fp8_norm_quant()
|
||||
|
||||
self.assertTrue(matching._fp8_img_attn_norm_quant)
|
||||
self.assertFalse(mismatched._fp8_img_attn_norm_quant)
|
||||
|
||||
def test_merged_qkv_uses_its_materialized_input_scale(
|
||||
self, _is_available, _capability
|
||||
) -> None:
|
||||
block = _block(_attention(fused=True, scales=(0.25,)))
|
||||
|
||||
block.configure_fp8_norm_quant()
|
||||
|
||||
self.assertTrue(block._fp8_img_attn_norm_quant)
|
||||
|
||||
def test_nonpositive_scale_keeps_fusion_disabled(
|
||||
self, _is_available, _capability
|
||||
) -> None:
|
||||
block = _block(_attention(fused=True, scales=(0.0,)))
|
||||
|
||||
block.configure_fp8_norm_quant()
|
||||
|
||||
self.assertFalse(block._fp8_img_attn_norm_quant)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user