diff --git a/python/sglang/kernels/ops/diffusion/activation/silu_mul_bitexact.py b/python/sglang/kernels/ops/diffusion/activation/silu_mul_bitexact.py index 273789390d..2c767129e7 100644 --- a/python/sglang/kernels/ops/diffusion/activation/silu_mul_bitexact.py +++ b/python/sglang/kernels/ops/diffusion/activation/silu_mul_bitexact.py @@ -47,6 +47,7 @@ def _packed_silu_mul_kernel( x_ptr, num_rows, row_stride, + out_row_stride, D: tl.constexpr, BLOCK: tl.constexpr, ): @@ -58,7 +59,7 @@ def _packed_silu_mul_kernel( a = tl.load(x_ptr + row_base + cols, mask=mask, other=0.0).to(tl.float32) b = tl.load(x_ptr + row_base + D + cols, mask=mask, other=0.0).to(tl.float32) s = round_bf16_to_fp32(a * tl.sigmoid(a)) - tl.store(out_ptr + row * D + cols, s * b, mask=mask) + tl.store(out_ptr + row * out_row_stride + cols, s * b, mask=mask) def _fake_silu_mul(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor: @@ -99,8 +100,14 @@ def fused_silu_mul_bitexact(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor: @replay_launches -def fused_packed_silu_mul_bitexact(x: torch.Tensor) -> torch.Tensor: - """Bit-exact SwiGLU over a contiguous packed ``[..., 2 * D]`` input.""" +def fused_packed_silu_mul_bitexact( + x: torch.Tensor, out: torch.Tensor | None = None +) -> torch.Tensor: + """Bit-exact SwiGLU over a contiguous packed ``[..., 2 * D]`` input. + + ``out`` may be a column slice of a wider buffer (row-strided ``[..., D]``), + so the result can land directly inside the next GEMM's input. + """ if not ( (x.is_cuda or x.is_xpu) and x.dtype is torch.bfloat16 @@ -115,13 +122,23 @@ def fused_packed_silu_mul_bitexact(x: torch.Tensor) -> torch.Tensor: hidden = x.shape[-1] // 2 rows = x.numel() // x.shape[-1] row_stride = x.stride(-2) - out = torch.empty((*x.shape[:-1], hidden), dtype=x.dtype, device=x.device) + if out is None: + out = torch.empty((*x.shape[:-1], hidden), dtype=x.dtype, device=x.device) + elif not ( + out.shape == (*x.shape[:-1], hidden) + and out.dtype is x.dtype + and out.device == x.device + and out.stride(-1) == 1 + and out.stride(0) == out.shape[1] * out.stride(1) + ): + raise RuntimeError("unsupported output for packed fused SiLU-mul") with torch.get_device_module(x.device).device(x.device): _packed_silu_mul_kernel[(rows, triton.cdiv(hidden, 1024))]( out, x, rows, row_stride, + out.stride(-2), D=hidden, BLOCK=1024, ) diff --git a/python/sglang/multimodal_gen/runtime/models/dits/flux_2.py b/python/sglang/multimodal_gen/runtime/models/dits/flux_2.py index ef27433f87..5157e001e5 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/flux_2.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/flux_2.py @@ -460,8 +460,21 @@ def _flux2_norm_modulate( ) -def _flux2_swiglu(x: torch.Tensor) -> torch.Tensor: - """Bit-exact fused SwiGLU for the packed FLUX.2 FFN projection.""" +def _flux2_swiglu(x: torch.Tensor, out: Optional[torch.Tensor] = None) -> torch.Tensor: + """Bit-exact fused SwiGLU for the packed FLUX.2 FFN projection. + + With ``out`` (e.g. a column slice of the next GEMM's input) the result is + written there and ``out`` is returned. + """ + if out is None: + return _flux2_swiglu_impl(x, None) + result = _flux2_swiglu_impl(x, out) + if result.data_ptr() != out.data_ptr(): + out.copy_(result) + return out + + +def _flux2_swiglu_impl(x: torch.Tensor, out: Optional[torch.Tensor]) -> torch.Tensor: half = x.shape[-1] // 2 # Let Inductor fuse the reference expression in torch.compile mode. if torch.compiler.is_compiling(): @@ -492,7 +505,7 @@ def _flux2_swiglu(x: torch.Tensor) -> torch.Tensor: return F.silu(x[..., :half]) * x[..., half:] if can_fuse: try: - out = fused_packed_silu_mul_bitexact(x) + out = fused_packed_silu_mul_bitexact(x, out) except Exception as exc: _FLUX2_SWIGLU.on_exception(exc, logger=logger) else: @@ -513,8 +526,10 @@ class Flux2SwiGLU(nn.Module): layer fused into the first linear layer of the FF sub-block. Thus, this module has no trainable parameters. """ - def forward(self, x: torch.Tensor) -> torch.Tensor: - return _flux2_swiglu(x) + def forward( + self, x: torch.Tensor, out: Optional[torch.Tensor] = None + ) -> torch.Tensor: + return _flux2_swiglu(x, out) class Flux2FeedForward(nn.Module): @@ -1101,6 +1116,19 @@ class Flux2ParallelSelfAttention(torch.nn.Module, AttentionModuleMixin): hidden_states = hidden_states.flatten(2, 3) hidden_states = hidden_states.to(query.dtype) + # BF16: build the output projection's [attn | mlp] input in place. SwiGLU + # writes its columns directly and only the narrower attention output is + # copied, instead of a full-width cat; the GEMM input is byte-identical. + if not (self._enable_fp8_token_cat or self._enable_nvfp4_token_cat): + attn_dim = hidden_states.shape[-1] + gemm_input = hidden_states.new_empty( + (*hidden_states.shape[:-1], attn_dim + mlp_hidden_states.shape[-1] // 2) + ) + self.mlp_act_fn(mlp_hidden_states, out=gemm_input[..., attn_dim:]) + gemm_input[..., :attn_dim].copy_(hidden_states) + hidden_states, _ = self.to_out(gemm_input) + return hidden_states + # Handle the feedforward (FF) logic mlp_hidden_states = self.mlp_act_fn(mlp_hidden_states) diff --git a/test/registered/kernels/ops/diffusion/test_flux2_xpu_norm_fusions.py b/test/registered/kernels/ops/diffusion/test_flux2_xpu_norm_fusions.py index cc9acf361a..d748313169 100644 --- a/test/registered/kernels/ops/diffusion/test_flux2_xpu_norm_fusions.py +++ b/test/registered/kernels/ops/diffusion/test_flux2_xpu_norm_fusions.py @@ -147,6 +147,21 @@ class TestFlux2XpuNormFusions(CustomTestCase): # Rows narrower than torch-xpu's 1024-wide work-group take another dispatch. self.assertFalse(can_use_xpu_layernorm_modulate(torch.bfloat16, 3072)) + def test_swiglu_into_out_projection_input_matches_cat(self): + # The single-stream block writes SwiGLU straight into the to_out input + # buffer (row stride 1536 + 4608); it must equal the old cat input. + torch.manual_seed(4) + attn = torch.randn(1, 64, 1536, device="xpu", dtype=torch.bfloat16) + mlp_proj = torch.randn(1, 64, 9216, device="xpu", dtype=torch.bfloat16) + expected = torch.cat([attn, flux2._flux2_swiglu(mlp_proj)], dim=-1) + + gemm_input = torch.full_like(expected, float("nan")) + returned = flux2._flux2_swiglu(mlp_proj, out=gemm_input[..., 1536:]) + gemm_input[..., :1536].copy_(attn) + + self.assertEqual(returned.data_ptr(), gemm_input[..., 1536:].data_ptr()) + self.assertTrue(torch.equal(gemm_input, expected)) + def test_launchers_reject_layouts_they_do_not_replicate(self): x, update, gate, scale, shift = _inputs(16, 1.0, 4) strided = x.transpose(1, 2).contiguous().transpose(1, 2)