[Perf] flux2: Build the single-block output projection without a cat (#41943)

Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Yihao Wang
2026-10-24 09:33:26 +08:00
committed by GitHub
co-authored by Claude Opus 5
parent 1d4dcc81cc
commit 5bef70c0a2
3 changed files with 69 additions and 9 deletions
@@ -47,6 +47,7 @@ def _packed_silu_mul_kernel(
x_ptr,
num_rows,
row_stride,
out_row_stride,
D: tl.constexpr,
BLOCK: tl.constexpr,
):
@@ -58,7 +59,7 @@ def _packed_silu_mul_kernel(
a = tl.load(x_ptr + row_base + cols, mask=mask, other=0.0).to(tl.float32)
b = tl.load(x_ptr + row_base + D + cols, mask=mask, other=0.0).to(tl.float32)
s = round_bf16_to_fp32(a * tl.sigmoid(a))
tl.store(out_ptr + row * D + cols, s * b, mask=mask)
tl.store(out_ptr + row * out_row_stride + cols, s * b, mask=mask)
def _fake_silu_mul(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor:
@@ -99,8 +100,14 @@ def fused_silu_mul_bitexact(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor:
@replay_launches
def fused_packed_silu_mul_bitexact(x: torch.Tensor) -> torch.Tensor:
"""Bit-exact SwiGLU over a contiguous packed ``[..., 2 * D]`` input."""
def fused_packed_silu_mul_bitexact(
x: torch.Tensor, out: torch.Tensor | None = None
) -> torch.Tensor:
"""Bit-exact SwiGLU over a contiguous packed ``[..., 2 * D]`` input.
``out`` may be a column slice of a wider buffer (row-strided ``[..., D]``),
so the result can land directly inside the next GEMM's input.
"""
if not (
(x.is_cuda or x.is_xpu)
and x.dtype is torch.bfloat16
@@ -115,13 +122,23 @@ def fused_packed_silu_mul_bitexact(x: torch.Tensor) -> torch.Tensor:
hidden = x.shape[-1] // 2
rows = x.numel() // x.shape[-1]
row_stride = x.stride(-2)
out = torch.empty((*x.shape[:-1], hidden), dtype=x.dtype, device=x.device)
if out is None:
out = torch.empty((*x.shape[:-1], hidden), dtype=x.dtype, device=x.device)
elif not (
out.shape == (*x.shape[:-1], hidden)
and out.dtype is x.dtype
and out.device == x.device
and out.stride(-1) == 1
and out.stride(0) == out.shape[1] * out.stride(1)
):
raise RuntimeError("unsupported output for packed fused SiLU-mul")
with torch.get_device_module(x.device).device(x.device):
_packed_silu_mul_kernel[(rows, triton.cdiv(hidden, 1024))](
out,
x,
rows,
row_stride,
out.stride(-2),
D=hidden,
BLOCK=1024,
)
@@ -460,8 +460,21 @@ def _flux2_norm_modulate(
)
def _flux2_swiglu(x: torch.Tensor) -> torch.Tensor:
"""Bit-exact fused SwiGLU for the packed FLUX.2 FFN projection."""
def _flux2_swiglu(x: torch.Tensor, out: Optional[torch.Tensor] = None) -> torch.Tensor:
"""Bit-exact fused SwiGLU for the packed FLUX.2 FFN projection.
With ``out`` (e.g. a column slice of the next GEMM's input) the result is
written there and ``out`` is returned.
"""
if out is None:
return _flux2_swiglu_impl(x, None)
result = _flux2_swiglu_impl(x, out)
if result.data_ptr() != out.data_ptr():
out.copy_(result)
return out
def _flux2_swiglu_impl(x: torch.Tensor, out: Optional[torch.Tensor]) -> torch.Tensor:
half = x.shape[-1] // 2
# Let Inductor fuse the reference expression in torch.compile mode.
if torch.compiler.is_compiling():
@@ -492,7 +505,7 @@ def _flux2_swiglu(x: torch.Tensor) -> torch.Tensor:
return F.silu(x[..., :half]) * x[..., half:]
if can_fuse:
try:
out = fused_packed_silu_mul_bitexact(x)
out = fused_packed_silu_mul_bitexact(x, out)
except Exception as exc:
_FLUX2_SWIGLU.on_exception(exc, logger=logger)
else:
@@ -513,8 +526,10 @@ class Flux2SwiGLU(nn.Module):
layer fused into the first linear layer of the FF sub-block. Thus, this module has no trainable parameters.
"""
def forward(self, x: torch.Tensor) -> torch.Tensor:
return _flux2_swiglu(x)
def forward(
self, x: torch.Tensor, out: Optional[torch.Tensor] = None
) -> torch.Tensor:
return _flux2_swiglu(x, out)
class Flux2FeedForward(nn.Module):
@@ -1101,6 +1116,19 @@ class Flux2ParallelSelfAttention(torch.nn.Module, AttentionModuleMixin):
hidden_states = hidden_states.flatten(2, 3)
hidden_states = hidden_states.to(query.dtype)
# BF16: build the output projection's [attn | mlp] input in place. SwiGLU
# writes its columns directly and only the narrower attention output is
# copied, instead of a full-width cat; the GEMM input is byte-identical.
if not (self._enable_fp8_token_cat or self._enable_nvfp4_token_cat):
attn_dim = hidden_states.shape[-1]
gemm_input = hidden_states.new_empty(
(*hidden_states.shape[:-1], attn_dim + mlp_hidden_states.shape[-1] // 2)
)
self.mlp_act_fn(mlp_hidden_states, out=gemm_input[..., attn_dim:])
gemm_input[..., :attn_dim].copy_(hidden_states)
hidden_states, _ = self.to_out(gemm_input)
return hidden_states
# Handle the feedforward (FF) logic
mlp_hidden_states = self.mlp_act_fn(mlp_hidden_states)
@@ -147,6 +147,21 @@ class TestFlux2XpuNormFusions(CustomTestCase):
# Rows narrower than torch-xpu's 1024-wide work-group take another dispatch.
self.assertFalse(can_use_xpu_layernorm_modulate(torch.bfloat16, 3072))
def test_swiglu_into_out_projection_input_matches_cat(self):
# The single-stream block writes SwiGLU straight into the to_out input
# buffer (row stride 1536 + 4608); it must equal the old cat input.
torch.manual_seed(4)
attn = torch.randn(1, 64, 1536, device="xpu", dtype=torch.bfloat16)
mlp_proj = torch.randn(1, 64, 9216, device="xpu", dtype=torch.bfloat16)
expected = torch.cat([attn, flux2._flux2_swiglu(mlp_proj)], dim=-1)
gemm_input = torch.full_like(expected, float("nan"))
returned = flux2._flux2_swiglu(mlp_proj, out=gemm_input[..., 1536:])
gemm_input[..., :1536].copy_(attn)
self.assertEqual(returned.data_ptr(), gemm_input[..., 1536:].data_ptr())
self.assertTrue(torch.equal(gemm_input, expected))
def test_launchers_reject_layouts_they_do_not_replicate(self):
x, update, gate, scale, shift = _inputs(16, 1.0, 4)
strided = x.transpose(1, 2).contiguous().transpose(1, 2)