[Perf] flux2: Build the single-block output projection without a cat (#41943)
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
1d4dcc81cc
commit
5bef70c0a2
@@ -47,6 +47,7 @@ def _packed_silu_mul_kernel(
|
||||
x_ptr,
|
||||
num_rows,
|
||||
row_stride,
|
||||
out_row_stride,
|
||||
D: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
@@ -58,7 +59,7 @@ def _packed_silu_mul_kernel(
|
||||
a = tl.load(x_ptr + row_base + cols, mask=mask, other=0.0).to(tl.float32)
|
||||
b = tl.load(x_ptr + row_base + D + cols, mask=mask, other=0.0).to(tl.float32)
|
||||
s = round_bf16_to_fp32(a * tl.sigmoid(a))
|
||||
tl.store(out_ptr + row * D + cols, s * b, mask=mask)
|
||||
tl.store(out_ptr + row * out_row_stride + cols, s * b, mask=mask)
|
||||
|
||||
|
||||
def _fake_silu_mul(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor:
|
||||
@@ -99,8 +100,14 @@ def fused_silu_mul_bitexact(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor:
|
||||
|
||||
|
||||
@replay_launches
|
||||
def fused_packed_silu_mul_bitexact(x: torch.Tensor) -> torch.Tensor:
|
||||
"""Bit-exact SwiGLU over a contiguous packed ``[..., 2 * D]`` input."""
|
||||
def fused_packed_silu_mul_bitexact(
|
||||
x: torch.Tensor, out: torch.Tensor | None = None
|
||||
) -> torch.Tensor:
|
||||
"""Bit-exact SwiGLU over a contiguous packed ``[..., 2 * D]`` input.
|
||||
|
||||
``out`` may be a column slice of a wider buffer (row-strided ``[..., D]``),
|
||||
so the result can land directly inside the next GEMM's input.
|
||||
"""
|
||||
if not (
|
||||
(x.is_cuda or x.is_xpu)
|
||||
and x.dtype is torch.bfloat16
|
||||
@@ -115,13 +122,23 @@ def fused_packed_silu_mul_bitexact(x: torch.Tensor) -> torch.Tensor:
|
||||
hidden = x.shape[-1] // 2
|
||||
rows = x.numel() // x.shape[-1]
|
||||
row_stride = x.stride(-2)
|
||||
out = torch.empty((*x.shape[:-1], hidden), dtype=x.dtype, device=x.device)
|
||||
if out is None:
|
||||
out = torch.empty((*x.shape[:-1], hidden), dtype=x.dtype, device=x.device)
|
||||
elif not (
|
||||
out.shape == (*x.shape[:-1], hidden)
|
||||
and out.dtype is x.dtype
|
||||
and out.device == x.device
|
||||
and out.stride(-1) == 1
|
||||
and out.stride(0) == out.shape[1] * out.stride(1)
|
||||
):
|
||||
raise RuntimeError("unsupported output for packed fused SiLU-mul")
|
||||
with torch.get_device_module(x.device).device(x.device):
|
||||
_packed_silu_mul_kernel[(rows, triton.cdiv(hidden, 1024))](
|
||||
out,
|
||||
x,
|
||||
rows,
|
||||
row_stride,
|
||||
out.stride(-2),
|
||||
D=hidden,
|
||||
BLOCK=1024,
|
||||
)
|
||||
|
||||
@@ -460,8 +460,21 @@ def _flux2_norm_modulate(
|
||||
)
|
||||
|
||||
|
||||
def _flux2_swiglu(x: torch.Tensor) -> torch.Tensor:
|
||||
"""Bit-exact fused SwiGLU for the packed FLUX.2 FFN projection."""
|
||||
def _flux2_swiglu(x: torch.Tensor, out: Optional[torch.Tensor] = None) -> torch.Tensor:
|
||||
"""Bit-exact fused SwiGLU for the packed FLUX.2 FFN projection.
|
||||
|
||||
With ``out`` (e.g. a column slice of the next GEMM's input) the result is
|
||||
written there and ``out`` is returned.
|
||||
"""
|
||||
if out is None:
|
||||
return _flux2_swiglu_impl(x, None)
|
||||
result = _flux2_swiglu_impl(x, out)
|
||||
if result.data_ptr() != out.data_ptr():
|
||||
out.copy_(result)
|
||||
return out
|
||||
|
||||
|
||||
def _flux2_swiglu_impl(x: torch.Tensor, out: Optional[torch.Tensor]) -> torch.Tensor:
|
||||
half = x.shape[-1] // 2
|
||||
# Let Inductor fuse the reference expression in torch.compile mode.
|
||||
if torch.compiler.is_compiling():
|
||||
@@ -492,7 +505,7 @@ def _flux2_swiglu(x: torch.Tensor) -> torch.Tensor:
|
||||
return F.silu(x[..., :half]) * x[..., half:]
|
||||
if can_fuse:
|
||||
try:
|
||||
out = fused_packed_silu_mul_bitexact(x)
|
||||
out = fused_packed_silu_mul_bitexact(x, out)
|
||||
except Exception as exc:
|
||||
_FLUX2_SWIGLU.on_exception(exc, logger=logger)
|
||||
else:
|
||||
@@ -513,8 +526,10 @@ class Flux2SwiGLU(nn.Module):
|
||||
layer fused into the first linear layer of the FF sub-block. Thus, this module has no trainable parameters.
|
||||
"""
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
return _flux2_swiglu(x)
|
||||
def forward(
|
||||
self, x: torch.Tensor, out: Optional[torch.Tensor] = None
|
||||
) -> torch.Tensor:
|
||||
return _flux2_swiglu(x, out)
|
||||
|
||||
|
||||
class Flux2FeedForward(nn.Module):
|
||||
@@ -1101,6 +1116,19 @@ class Flux2ParallelSelfAttention(torch.nn.Module, AttentionModuleMixin):
|
||||
hidden_states = hidden_states.flatten(2, 3)
|
||||
hidden_states = hidden_states.to(query.dtype)
|
||||
|
||||
# BF16: build the output projection's [attn | mlp] input in place. SwiGLU
|
||||
# writes its columns directly and only the narrower attention output is
|
||||
# copied, instead of a full-width cat; the GEMM input is byte-identical.
|
||||
if not (self._enable_fp8_token_cat or self._enable_nvfp4_token_cat):
|
||||
attn_dim = hidden_states.shape[-1]
|
||||
gemm_input = hidden_states.new_empty(
|
||||
(*hidden_states.shape[:-1], attn_dim + mlp_hidden_states.shape[-1] // 2)
|
||||
)
|
||||
self.mlp_act_fn(mlp_hidden_states, out=gemm_input[..., attn_dim:])
|
||||
gemm_input[..., :attn_dim].copy_(hidden_states)
|
||||
hidden_states, _ = self.to_out(gemm_input)
|
||||
return hidden_states
|
||||
|
||||
# Handle the feedforward (FF) logic
|
||||
mlp_hidden_states = self.mlp_act_fn(mlp_hidden_states)
|
||||
|
||||
|
||||
@@ -147,6 +147,21 @@ class TestFlux2XpuNormFusions(CustomTestCase):
|
||||
# Rows narrower than torch-xpu's 1024-wide work-group take another dispatch.
|
||||
self.assertFalse(can_use_xpu_layernorm_modulate(torch.bfloat16, 3072))
|
||||
|
||||
def test_swiglu_into_out_projection_input_matches_cat(self):
|
||||
# The single-stream block writes SwiGLU straight into the to_out input
|
||||
# buffer (row stride 1536 + 4608); it must equal the old cat input.
|
||||
torch.manual_seed(4)
|
||||
attn = torch.randn(1, 64, 1536, device="xpu", dtype=torch.bfloat16)
|
||||
mlp_proj = torch.randn(1, 64, 9216, device="xpu", dtype=torch.bfloat16)
|
||||
expected = torch.cat([attn, flux2._flux2_swiglu(mlp_proj)], dim=-1)
|
||||
|
||||
gemm_input = torch.full_like(expected, float("nan"))
|
||||
returned = flux2._flux2_swiglu(mlp_proj, out=gemm_input[..., 1536:])
|
||||
gemm_input[..., :1536].copy_(attn)
|
||||
|
||||
self.assertEqual(returned.data_ptr(), gemm_input[..., 1536:].data_ptr())
|
||||
self.assertTrue(torch.equal(gemm_input, expected))
|
||||
|
||||
def test_launchers_reject_layouts_they_do_not_replicate(self):
|
||||
x, update, gate, scale, shift = _inputs(16, 1.0, 4)
|
||||
strided = x.transpose(1, 2).contiguous().transpose(1, 2)
|
||||
|
||||
Reference in New Issue
Block a user