[diffusion] fix: fix CFG normalization to use per-sample norm instead of batch-flattened norm (#41706)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
Avaya Aggarwal
2026-10-09 16:43:18 +08:00
committed by GitHub
co-authored by Mick Qian
parent 3ab142adc4
commit d677848968
2 changed files with 80 additions and 6 deletions
@@ -141,12 +141,14 @@ def _apply_cfg_normalization(
) -> torch.Tensor:
cond_f = noise_pred_cond.float()
pred_f = noise_pred.float()
ori_norm = torch.linalg.vector_norm(cond_f)
new_norm = torch.linalg.vector_norm(pred_f)
dims = list(range(1, noise_pred.ndim))
ori_norm = torch.linalg.vector_norm(cond_f, dim=dims, keepdim=True)
new_norm = torch.linalg.vector_norm(pred_f, dim=dims, keepdim=True)
max_norm = ori_norm * cfg_normalization
if new_norm > max_norm:
noise_pred = noise_pred * (max_norm / new_norm)
return noise_pred
scale = torch.where(
new_norm > max_norm, max_norm / new_norm, torch.ones_like(new_norm)
)
return (noise_pred * scale).to(noise_pred.dtype)
def _rescale_noise_cfg(
@@ -3,7 +3,10 @@ from unittest.mock import MagicMock
import torch
from sglang.multimodal_gen.runtime.distributed.cfg_policy import CFGPolicy
from sglang.multimodal_gen.runtime.distributed.cfg_policy import (
CFGPolicy,
_apply_cfg_normalization,
)
class TestCFGPolicyCombine(unittest.TestCase):
@@ -29,5 +32,74 @@ class TestCFGPolicyCombine(unittest.TestCase):
self.assertFalse(torch.equal(serial, parallel))
class TestCFGNormalizationDtype(unittest.TestCase):
def test_normalization_preserves_prediction_dtype(self):
"""Per-sample fp32 scale must not promote a bf16/fp16 prediction to fp32."""
for dtype in (torch.float32, torch.float16, torch.bfloat16):
for pred_value, cond_value in ((0.25, 1.0), (1.0, 0.25)):
with self.subTest(dtype=dtype, pred=pred_value):
pred = torch.full((2, 2, 3), pred_value, dtype=dtype)
cond = torch.full((2, 2, 3), cond_value, dtype=dtype)
out = _apply_cfg_normalization(pred, cond, 0.7)
self.assertEqual(out.dtype, dtype)
class TestCFGNormalizationPerSample(unittest.TestCase):
"""CFG normalization clamps each sample by its own norm, not the batch's.
A batch-flattened norm makes one request's output depend on whichever
requests happen to be co-scheduled with it in the same DiT forward pass.
"""
CFG_NORMALIZATION = 0.7
@staticmethod
def _batch(seed):
generator = torch.Generator().manual_seed(seed)
pred = torch.randn(4, 3, 8, 8, generator=generator)
cond = torch.randn(4, 3, 8, 8, generator=generator) * 0.3
return pred, cond
@staticmethod
def _per_sample_norm(tensor):
dims = list(range(1, tensor.ndim))
return torch.linalg.vector_norm(tensor.float(), dim=dims)
def test_sample_result_is_independent_of_batch_composition(self):
pred, cond = self._batch(seed=0)
pred[1] *= 50.0
co_batched = _apply_cfg_normalization(pred, cond, self.CFG_NORMALIZATION)
solo = _apply_cfg_normalization(pred[:1], cond[:1], self.CFG_NORMALIZATION)
self.assertTrue(torch.equal(solo, co_batched[:1]))
def test_every_sample_respects_its_own_threshold(self):
pred, cond = self._batch(seed=1)
pred[0] *= 100.0
out = _apply_cfg_normalization(pred, cond, self.CFG_NORMALIZATION)
ratio = self._per_sample_norm(out) / self._per_sample_norm(cond)
self.assertTrue(
torch.all(ratio <= self.CFG_NORMALIZATION + 1e-5),
f"per-sample norm ratios exceed threshold: {ratio.tolist()}",
)
def test_batch_size_one_matches_global_norm_reference(self):
"""At batch size 1 the per-sample norm must stay bit-identical to upstream."""
pred, cond = self._batch(seed=2)
pred, cond = pred[:1], cond[:1]
out = _apply_cfg_normalization(pred, cond, self.CFG_NORMALIZATION)
ori_norm = torch.linalg.vector_norm(cond.float())
new_norm = torch.linalg.vector_norm(pred.float())
max_norm = ori_norm * self.CFG_NORMALIZATION
expected = pred * (max_norm / new_norm) if new_norm > max_norm else pred
self.assertTrue(torch.equal(out, expected))
if __name__ == "__main__":
unittest.main()