diff --git a/python/sglang/multimodal_gen/runtime/distributed/cfg_policy.py b/python/sglang/multimodal_gen/runtime/distributed/cfg_policy.py index fb3c000d71..14965f3f83 100644 --- a/python/sglang/multimodal_gen/runtime/distributed/cfg_policy.py +++ b/python/sglang/multimodal_gen/runtime/distributed/cfg_policy.py @@ -141,12 +141,14 @@ def _apply_cfg_normalization( ) -> torch.Tensor: cond_f = noise_pred_cond.float() pred_f = noise_pred.float() - ori_norm = torch.linalg.vector_norm(cond_f) - new_norm = torch.linalg.vector_norm(pred_f) + dims = list(range(1, noise_pred.ndim)) + ori_norm = torch.linalg.vector_norm(cond_f, dim=dims, keepdim=True) + new_norm = torch.linalg.vector_norm(pred_f, dim=dims, keepdim=True) max_norm = ori_norm * cfg_normalization - if new_norm > max_norm: - noise_pred = noise_pred * (max_norm / new_norm) - return noise_pred + scale = torch.where( + new_norm > max_norm, max_norm / new_norm, torch.ones_like(new_norm) + ) + return (noise_pred * scale).to(noise_pred.dtype) def _rescale_noise_cfg( diff --git a/python/sglang/multimodal_gen/test/unit/test_cfg_policy.py b/python/sglang/multimodal_gen/test/unit/test_cfg_policy.py index 09d48eaada..3eb6b250d1 100644 --- a/python/sglang/multimodal_gen/test/unit/test_cfg_policy.py +++ b/python/sglang/multimodal_gen/test/unit/test_cfg_policy.py @@ -3,7 +3,10 @@ from unittest.mock import MagicMock import torch -from sglang.multimodal_gen.runtime.distributed.cfg_policy import CFGPolicy +from sglang.multimodal_gen.runtime.distributed.cfg_policy import ( + CFGPolicy, + _apply_cfg_normalization, +) class TestCFGPolicyCombine(unittest.TestCase): @@ -29,5 +32,74 @@ class TestCFGPolicyCombine(unittest.TestCase): self.assertFalse(torch.equal(serial, parallel)) +class TestCFGNormalizationDtype(unittest.TestCase): + def test_normalization_preserves_prediction_dtype(self): + """Per-sample fp32 scale must not promote a bf16/fp16 prediction to fp32.""" + for dtype in (torch.float32, torch.float16, torch.bfloat16): + for pred_value, cond_value in ((0.25, 1.0), (1.0, 0.25)): + with self.subTest(dtype=dtype, pred=pred_value): + pred = torch.full((2, 2, 3), pred_value, dtype=dtype) + cond = torch.full((2, 2, 3), cond_value, dtype=dtype) + out = _apply_cfg_normalization(pred, cond, 0.7) + self.assertEqual(out.dtype, dtype) + + +class TestCFGNormalizationPerSample(unittest.TestCase): + """CFG normalization clamps each sample by its own norm, not the batch's. + + A batch-flattened norm makes one request's output depend on whichever + requests happen to be co-scheduled with it in the same DiT forward pass. + """ + + CFG_NORMALIZATION = 0.7 + + @staticmethod + def _batch(seed): + generator = torch.Generator().manual_seed(seed) + pred = torch.randn(4, 3, 8, 8, generator=generator) + cond = torch.randn(4, 3, 8, 8, generator=generator) * 0.3 + return pred, cond + + @staticmethod + def _per_sample_norm(tensor): + dims = list(range(1, tensor.ndim)) + return torch.linalg.vector_norm(tensor.float(), dim=dims) + + def test_sample_result_is_independent_of_batch_composition(self): + pred, cond = self._batch(seed=0) + pred[1] *= 50.0 + + co_batched = _apply_cfg_normalization(pred, cond, self.CFG_NORMALIZATION) + solo = _apply_cfg_normalization(pred[:1], cond[:1], self.CFG_NORMALIZATION) + + self.assertTrue(torch.equal(solo, co_batched[:1])) + + def test_every_sample_respects_its_own_threshold(self): + pred, cond = self._batch(seed=1) + pred[0] *= 100.0 + + out = _apply_cfg_normalization(pred, cond, self.CFG_NORMALIZATION) + + ratio = self._per_sample_norm(out) / self._per_sample_norm(cond) + self.assertTrue( + torch.all(ratio <= self.CFG_NORMALIZATION + 1e-5), + f"per-sample norm ratios exceed threshold: {ratio.tolist()}", + ) + + def test_batch_size_one_matches_global_norm_reference(self): + """At batch size 1 the per-sample norm must stay bit-identical to upstream.""" + pred, cond = self._batch(seed=2) + pred, cond = pred[:1], cond[:1] + + out = _apply_cfg_normalization(pred, cond, self.CFG_NORMALIZATION) + + ori_norm = torch.linalg.vector_norm(cond.float()) + new_norm = torch.linalg.vector_norm(pred.float()) + max_norm = ori_norm * self.CFG_NORMALIZATION + expected = pred * (max_norm / new_norm) if new_norm > max_norm else pred + + self.assertTrue(torch.equal(out, expected)) + + if __name__ == "__main__": unittest.main()