[Diffusion] Accelerate Cosmos3 Edge on Hopper with lossless fusions (#40386)
This commit is contained in:
@@ -134,10 +134,10 @@ def run_activation(
|
||||
|
||||
@register_custom_op(mutates_args=["out"])
|
||||
def _run_unary_activation_inplace(
|
||||
op_name: str, input: torch.Tensor, out: torch.Tensor
|
||||
op_name: str, input: torch.Tensor, out: torch.Tensor, fast_math: bool = True
|
||||
) -> None:
|
||||
last = input.shape[-1]
|
||||
module = activation_module(input.dtype)
|
||||
module = activation_module(input.dtype, fast_math=fast_math)
|
||||
module.run_unary_activation(input.view(-1, last), out.view(-1, last), op_name)
|
||||
|
||||
|
||||
@@ -145,6 +145,8 @@ def run_unary_activation(
|
||||
op_name: str,
|
||||
input: torch.Tensor,
|
||||
out: Optional[torch.Tensor] = None,
|
||||
*,
|
||||
fast_math: bool = True,
|
||||
) -> torch.Tensor:
|
||||
"""Apply a standalone (non-gated) element-wise activation: ``out = act(input)``.
|
||||
|
||||
@@ -156,16 +158,18 @@ def run_unary_activation(
|
||||
)
|
||||
if out is None:
|
||||
out = torch.empty_like(input)
|
||||
_run_unary_activation_inplace(op_name, input, out)
|
||||
_run_unary_activation_inplace(op_name, input, out, fast_math)
|
||||
return out
|
||||
|
||||
|
||||
def relu2(
|
||||
input: torch.Tensor,
|
||||
out: Optional[torch.Tensor] = None,
|
||||
*,
|
||||
fast_math: bool = True,
|
||||
) -> torch.Tensor:
|
||||
"""Squared ReLU: ``out = max(0, input) ** 2`` (element-wise)."""
|
||||
return run_unary_activation("relu2", input, out)
|
||||
"""Squared ReLU; disable fast math to preserve BF16 subnormal results."""
|
||||
return run_unary_activation("relu2", input, out, fast_math=fast_math)
|
||||
|
||||
|
||||
def silu_and_mul(
|
||||
|
||||
@@ -14,6 +14,7 @@ import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
from sglang.kernels.ops.activation.activation import relu2
|
||||
from sglang.kernels.ops.diffusion import (
|
||||
can_use_fused_inplace_qknorm_rope,
|
||||
fused_qknorm_rope_pack_kv,
|
||||
@@ -81,12 +82,18 @@ def _can_enable_t1_fused_qk_norm_rope(
|
||||
tp_size: int,
|
||||
sp_size: int,
|
||||
is_compiled: bool,
|
||||
hidden_size: int = 0,
|
||||
) -> bool:
|
||||
if is_compiled:
|
||||
return False
|
||||
if is_blackwell:
|
||||
return True
|
||||
return is_hopper and hidden_act != "relu2" and tp_size == 1 and sp_size == 1
|
||||
return (
|
||||
is_hopper
|
||||
and (hidden_act != "relu2" or hidden_size == 2048)
|
||||
and tp_size == 1
|
||||
and sp_size == 1
|
||||
)
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
@@ -540,8 +547,18 @@ class Cosmos3DenseMLP(nn.Module):
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
up, _ = self.up_proj(x)
|
||||
up = F.relu(up)
|
||||
out, _ = self.down_proj(up * up)
|
||||
if (
|
||||
up.is_cuda
|
||||
and up.dtype == torch.bfloat16
|
||||
and up.is_contiguous()
|
||||
and not torch.is_grad_enabled()
|
||||
and not torch.compiler.is_compiling()
|
||||
):
|
||||
up = relu2(up, fast_math=False)
|
||||
else:
|
||||
up = F.relu(up)
|
||||
up = up * up
|
||||
out, _ = self.down_proj(up)
|
||||
return out
|
||||
|
||||
|
||||
@@ -1710,13 +1727,14 @@ class Cosmos3OmniTransformer(CachableDiT, LayerwiseOffloadableModuleMixin):
|
||||
self._ensure_cache_dicts()
|
||||
|
||||
# The T=1 fused path is faster on Blackwell. It also benefits the
|
||||
# single-GPU Hopper Nano (SwiGLU) workload, while the Hopper
|
||||
# Cosmos3-Super (dense MLP) multi-GPU workload remains on the split
|
||||
# single-GPU Hopper Nano (SwiGLU) and Edge (2048-wide dense) workloads,
|
||||
# while the Hopper Cosmos3-Super (dense MLP) multi-GPU workload remains on the split
|
||||
# path because that shape regresses with the fusion.
|
||||
enable_t1_fused_qk_norm_rope = T == 1 and _can_enable_t1_fused_qk_norm_rope(
|
||||
is_blackwell=current_platform.is_blackwell(),
|
||||
is_hopper=current_platform.is_hopper(),
|
||||
hidden_act=self.hidden_act,
|
||||
hidden_size=self.hidden_size,
|
||||
tp_size=get_tp_world_size(),
|
||||
sp_size=get_sp_world_size(),
|
||||
is_compiled=self._gen_layers_torch_compiled,
|
||||
|
||||
@@ -112,6 +112,7 @@ class TestCosmos3T1FusedQKNormRoPE(unittest.TestCase):
|
||||
tp_size=1,
|
||||
sp_size=1,
|
||||
is_compiled=False,
|
||||
hidden_size=0,
|
||||
):
|
||||
return _can_enable_t1_fused_qk_norm_rope(
|
||||
is_blackwell=is_blackwell,
|
||||
@@ -120,6 +121,7 @@ class TestCosmos3T1FusedQKNormRoPE(unittest.TestCase):
|
||||
tp_size=tp_size,
|
||||
sp_size=sp_size,
|
||||
is_compiled=is_compiled,
|
||||
hidden_size=hidden_size,
|
||||
)
|
||||
|
||||
def test_blackwell_remains_enabled(self):
|
||||
@@ -138,6 +140,28 @@ class TestCosmos3T1FusedQKNormRoPE(unittest.TestCase):
|
||||
def test_hopper_dense_mlp_disabled(self):
|
||||
self.assertFalse(self._can_enable(is_hopper=True, hidden_act="relu2"))
|
||||
|
||||
def test_hopper_edge_single_gpu_enabled(self):
|
||||
self.assertTrue(
|
||||
self._can_enable(is_hopper=True, hidden_act="relu2", hidden_size=2048)
|
||||
)
|
||||
|
||||
def test_hopper_edge_parallel_and_compiled_disabled(self):
|
||||
for override in ({"tp_size": 2}, {"sp_size": 2}, {"is_compiled": True}):
|
||||
with self.subTest(override=override):
|
||||
self.assertFalse(
|
||||
self._can_enable(
|
||||
is_hopper=True,
|
||||
hidden_act="relu2",
|
||||
hidden_size=2048,
|
||||
**override,
|
||||
)
|
||||
)
|
||||
|
||||
def test_hopper_larger_dense_mlp_disabled(self):
|
||||
self.assertFalse(
|
||||
self._can_enable(is_hopper=True, hidden_act="relu2", hidden_size=4096)
|
||||
)
|
||||
|
||||
def test_hopper_tensor_parallel_disabled(self):
|
||||
self.assertFalse(self._can_enable(is_hopper=True, tp_size=2))
|
||||
|
||||
|
||||
Reference in New Issue
Block a user