From 2f5c9ac43d76eef11e8561c89c55a79db7db715a Mon Sep 17 00:00:00 2001 From: Xiaoyu Zhang <1182563586@qq.com> Date: Fri, 25 Sep 2026 18:01:06 +0800 Subject: [PATCH] [diffusion] Enable lossless Cosmos3 Super T2I QK fusion on Hopper TP2 (#40486) --- docs/docs/sglang-diffusion/fused_kernels.mdx | 3 ++- .../runtime/models/dits/cosmos3video.py | 15 +++++++------- .../multimodal_gen/test/unit/test_cosmos3.py | 20 ++++++++++++++++++- 3 files changed, 28 insertions(+), 10 deletions(-) diff --git a/docs/docs/sglang-diffusion/fused_kernels.mdx b/docs/docs/sglang-diffusion/fused_kernels.mdx index ce0052aa92..9ff506a7c4 100644 --- a/docs/docs/sglang-diffusion/fused_kernels.mdx +++ b/docs/docs/sglang-diffusion/fused_kernels.mdx @@ -234,7 +234,8 @@ Kernels are written against a specific eager chain in a specific model, so cover | SANA-Video | Packed QKV/KV; paired fp64 interleaved RoPE; GLUMB bias+SiLU / bias+GLU in eager and BCG; LN+modulate and residual-gate add during BCG; BF16-input linear attention at `quality=extra-high` or `quality=high` | | Sana-WM | bidirectional gated delta-net, fused QK inverse-RMS | | Wan / LongLive 2 VAE | temb table slices (Wan); VAE cat+pad and DupUp3D add; lossless FP32 normalization post-ops; full `channels_last_3d` RMSNorm+SiLU at `quality=extra-high` or `quality=high` | -| Cosmos3 / Krea2 / MiniMax-H3 | QK-norm + RoPE (Krea2 also CuTe-DSL norm+scale/shift; MiniMax-H3 also indexed modulation) | +| Cosmos3 | QK-norm + RoPE + KV packing; rounded T2I fusion also supports Hopper TP2/SP1 Super-Text2Image with the 5120-wide SwiGLU architecture | +| Krea2 / MiniMax-H3 | QK-norm + RoPE (Krea2 also CuTe-DSL norm+scale/shift; MiniMax-H3 also indexed modulation) | | FLUX.2 VAE / HunyuanVAE / latent upsampler | GroupNorm + SiLU (channels-last two-pass for FLUX.2) | ## Inspecting what is registered diff --git a/python/sglang/multimodal_gen/runtime/models/dits/cosmos3video.py b/python/sglang/multimodal_gen/runtime/models/dits/cosmos3video.py index cbfb04b4b4..019b6970d9 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/cosmos3video.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/cosmos3video.py @@ -88,12 +88,11 @@ def _can_enable_t1_fused_qk_norm_rope( return False if is_blackwell: return True - return ( - is_hopper - and (hidden_act != "relu2" or hidden_size == 2048) - and tp_size == 1 - and sp_size == 1 - ) + if not is_hopper or sp_size != 1: + return False + if tp_size == 1: + return hidden_act != "relu2" or hidden_size == 2048 + return tp_size == 2 and hidden_act == "silu" and hidden_size == 5120 # ----------------------------------------------------------------------------- @@ -1728,8 +1727,8 @@ class Cosmos3OmniTransformer(CachableDiT, LayerwiseOffloadableModuleMixin): # The T=1 fused path is faster on Blackwell. It also benefits the # single-GPU Hopper Nano (SwiGLU) and Edge (2048-wide dense) workloads, - # while the Hopper Cosmos3-Super (dense MLP) multi-GPU workload remains on the split - # path because that shape regresses with the fusion. + # as well as TP2 Super-Text2Image (SwiGLU). The older dense-MLP Super + # workload remains on the split path because that shape regresses. enable_t1_fused_qk_norm_rope = T == 1 and _can_enable_t1_fused_qk_norm_rope( is_blackwell=current_platform.is_blackwell(), is_hopper=current_platform.is_hopper(), diff --git a/python/sglang/multimodal_gen/test/unit/test_cosmos3.py b/python/sglang/multimodal_gen/test/unit/test_cosmos3.py index 79b8d45726..993bf3a38c 100644 --- a/python/sglang/multimodal_gen/test/unit/test_cosmos3.py +++ b/python/sglang/multimodal_gen/test/unit/test_cosmos3.py @@ -140,6 +140,24 @@ class TestCosmos3T1FusedQKNormRoPE(unittest.TestCase): def test_hopper_dense_mlp_disabled(self): self.assertFalse(self._can_enable(is_hopper=True, hidden_act="relu2")) + def test_hopper_super_t2i_tp2_enabled(self): + self.assertTrue(self._can_enable(is_hopper=True, hidden_size=5120, tp_size=2)) + + def test_hopper_super_t2i_unsupported_topologies_disabled(self): + for overrides in ( + {"hidden_act": "relu2"}, + {"hidden_act": "gelu"}, + {"hidden_size": 4096}, + {"tp_size": 4}, + {"sp_size": 2}, + {"is_compiled": True}, + {"is_hopper": False}, + ): + with self.subTest(overrides=overrides): + settings = dict(is_hopper=True, hidden_size=5120, tp_size=2) + settings.update(overrides) + self.assertFalse(self._can_enable(**settings)) + def test_hopper_edge_single_gpu_enabled(self): self.assertTrue( self._can_enable(is_hopper=True, hidden_act="relu2", hidden_size=2048) @@ -162,7 +180,7 @@ class TestCosmos3T1FusedQKNormRoPE(unittest.TestCase): self._can_enable(is_hopper=True, hidden_act="relu2", hidden_size=4096) ) - def test_hopper_tensor_parallel_disabled(self): + def test_hopper_other_tensor_parallel_shapes_disabled(self): self.assertFalse(self._can_enable(is_hopper=True, tp_size=2)) def test_hopper_sequence_parallel_disabled(self):