[Diffusion] Accelerate Cosmos3 Edge on Hopper with lossless fusions (#40386)

This commit is contained in:
Xiaoyu Zhang
2026-09-23 21:16:58 +08:00
committed by GitHub
parent 16e353d93e
commit 172b1b4825
5 changed files with 254 additions and 10 deletions
@@ -134,10 +134,10 @@ def run_activation(
@register_custom_op(mutates_args=["out"])
def _run_unary_activation_inplace(
op_name: str, input: torch.Tensor, out: torch.Tensor
op_name: str, input: torch.Tensor, out: torch.Tensor, fast_math: bool = True
) -> None:
last = input.shape[-1]
module = activation_module(input.dtype)
module = activation_module(input.dtype, fast_math=fast_math)
module.run_unary_activation(input.view(-1, last), out.view(-1, last), op_name)
@@ -145,6 +145,8 @@ def run_unary_activation(
op_name: str,
input: torch.Tensor,
out: Optional[torch.Tensor] = None,
*,
fast_math: bool = True,
) -> torch.Tensor:
"""Apply a standalone (non-gated) element-wise activation: ``out = act(input)``.
@@ -156,16 +158,18 @@ def run_unary_activation(
)
if out is None:
out = torch.empty_like(input)
_run_unary_activation_inplace(op_name, input, out)
_run_unary_activation_inplace(op_name, input, out, fast_math)
return out
def relu2(
input: torch.Tensor,
out: Optional[torch.Tensor] = None,
*,
fast_math: bool = True,
) -> torch.Tensor:
"""Squared ReLU: ``out = max(0, input) ** 2`` (element-wise)."""
return run_unary_activation("relu2", input, out)
"""Squared ReLU; disable fast math to preserve BF16 subnormal results."""
return run_unary_activation("relu2", input, out, fast_math=fast_math)
def silu_and_mul(
@@ -14,6 +14,7 @@ import torch
import torch.nn as nn
import torch.nn.functional as F
from sglang.kernels.ops.activation.activation import relu2
from sglang.kernels.ops.diffusion import (
can_use_fused_inplace_qknorm_rope,
fused_qknorm_rope_pack_kv,
@@ -81,12 +82,18 @@ def _can_enable_t1_fused_qk_norm_rope(
tp_size: int,
sp_size: int,
is_compiled: bool,
hidden_size: int = 0,
) -> bool:
if is_compiled:
return False
if is_blackwell:
return True
return is_hopper and hidden_act != "relu2" and tp_size == 1 and sp_size == 1
return (
is_hopper
and (hidden_act != "relu2" or hidden_size == 2048)
and tp_size == 1
and sp_size == 1
)
# -----------------------------------------------------------------------------
@@ -540,8 +547,18 @@ class Cosmos3DenseMLP(nn.Module):
def forward(self, x: torch.Tensor) -> torch.Tensor:
up, _ = self.up_proj(x)
up = F.relu(up)
out, _ = self.down_proj(up * up)
if (
up.is_cuda
and up.dtype == torch.bfloat16
and up.is_contiguous()
and not torch.is_grad_enabled()
and not torch.compiler.is_compiling()
):
up = relu2(up, fast_math=False)
else:
up = F.relu(up)
up = up * up
out, _ = self.down_proj(up)
return out
@@ -1710,13 +1727,14 @@ class Cosmos3OmniTransformer(CachableDiT, LayerwiseOffloadableModuleMixin):
self._ensure_cache_dicts()
# The T=1 fused path is faster on Blackwell. It also benefits the
# single-GPU Hopper Nano (SwiGLU) workload, while the Hopper
# Cosmos3-Super (dense MLP) multi-GPU workload remains on the split
# single-GPU Hopper Nano (SwiGLU) and Edge (2048-wide dense) workloads,
# while the Hopper Cosmos3-Super (dense MLP) multi-GPU workload remains on the split
# path because that shape regresses with the fusion.
enable_t1_fused_qk_norm_rope = T == 1 and _can_enable_t1_fused_qk_norm_rope(
is_blackwell=current_platform.is_blackwell(),
is_hopper=current_platform.is_hopper(),
hidden_act=self.hidden_act,
hidden_size=self.hidden_size,
tp_size=get_tp_world_size(),
sp_size=get_sp_world_size(),
is_compiled=self._gen_layers_torch_compiled,
@@ -112,6 +112,7 @@ class TestCosmos3T1FusedQKNormRoPE(unittest.TestCase):
tp_size=1,
sp_size=1,
is_compiled=False,
hidden_size=0,
):
return _can_enable_t1_fused_qk_norm_rope(
is_blackwell=is_blackwell,
@@ -120,6 +121,7 @@ class TestCosmos3T1FusedQKNormRoPE(unittest.TestCase):
tp_size=tp_size,
sp_size=sp_size,
is_compiled=is_compiled,
hidden_size=hidden_size,
)
def test_blackwell_remains_enabled(self):
@@ -138,6 +140,28 @@ class TestCosmos3T1FusedQKNormRoPE(unittest.TestCase):
def test_hopper_dense_mlp_disabled(self):
self.assertFalse(self._can_enable(is_hopper=True, hidden_act="relu2"))
def test_hopper_edge_single_gpu_enabled(self):
self.assertTrue(
self._can_enable(is_hopper=True, hidden_act="relu2", hidden_size=2048)
)
def test_hopper_edge_parallel_and_compiled_disabled(self):
for override in ({"tp_size": 2}, {"sp_size": 2}, {"is_compiled": True}):
with self.subTest(override=override):
self.assertFalse(
self._can_enable(
is_hopper=True,
hidden_act="relu2",
hidden_size=2048,
**override,
)
)
def test_hopper_larger_dense_mlp_disabled(self):
self.assertFalse(
self._can_enable(is_hopper=True, hidden_act="relu2", hidden_size=4096)
)
def test_hopper_tensor_parallel_disabled(self):
self.assertFalse(self._can_enable(is_hopper=True, tp_size=2))