[Diffusion] Fuse LongCat GELU+cat and support Edit-Turbo BCG (#40384)

This commit is contained in:
Xiaoyu Zhang
2026-09-24 17:27:56 +08:00
committed by GitHub
parent 174a5f37b9
commit 261cb826e6
10 changed files with 321 additions and 6 deletions
@@ -0,0 +1,90 @@
#include <sgl_kernel/tensor.h>
#include <sgl_kernel/runtime.cuh>
#include <sgl_kernel/type.cuh>
#include <sgl_kernel/utils.cuh>
#include <sgl_kernel/vec.cuh>
#include <tvm/ffi/container/tensor.h>
#include <algorithm>
#include <cmath>
#include <cstdint>
#include <limits>
namespace sglang {
template <typename T, int kVecN>
__global__ void gelu_tanh_cat_kernel(
const T* __restrict__ attn,
const T* __restrict__ mlp,
T* __restrict__ output,
uint32_t num_vecs,
uint32_t attn_vecs,
uint32_t mlp_vecs) {
using vec_t = device::AlignedVector<T, kVecN>;
const uint32_t row_vecs = attn_vecs + mlp_vecs;
const uint32_t stride = blockDim.x * gridDim.x;
for (uint32_t i = blockIdx.x * blockDim.x + threadIdx.x; i < num_vecs; i += stride) {
const uint32_t row = i / row_vecs;
const uint32_t col = i % row_vecs;
vec_t value;
if (col < attn_vecs) {
value.load(attn, row * attn_vecs + col);
} else {
value.load(mlp, row * mlp_vecs + col - attn_vecs);
#pragma unroll
for (int j = 0; j < kVecN; ++j) {
const float x = device::cast<fp32_t>(value[j]);
// Match aten's tanh GELU operation order, including the BF16 store
// that precedes the eager concatenation.
constexpr float kBeta = 0.7978845608028654f;
constexpr float kKappa = 0.044715f;
const float cube = x * x * x;
const float inner = kBeta * (x + kKappa * cube);
value[j] = device::cast<T>(0.5f * x * (1.0f + tanhf(inner)));
}
}
value.store(output, i);
}
}
template <typename T>
void gelu_tanh_cat(tvm::ffi::TensorView attn, tvm::ffi::TensorView mlp, tvm::ffi::TensorView output) {
using namespace host;
auto rows = SymbolicSize{"rows"};
auto attn_width = SymbolicSize{"attn_width"};
auto mlp_width = SymbolicSize{"mlp_width"};
auto device_ = SymbolicDevice{};
device_.set_options<kDLCUDA>();
TensorMatcher({rows, attn_width}).with_dtype<T>().with_device(device_).verify(attn);
TensorMatcher({rows, mlp_width}).with_dtype<T>().with_device(device_).verify(mlp);
const int64_t a = attn_width.unwrap();
const int64_t m = mlp_width.unwrap();
TensorMatcher({rows, a + m}).with_dtype<T>().with_device(device_).verify(output);
constexpr int kVecN = 16 / sizeof(T);
CHECK_HOST(a > 0 && m > 0 && a % kVecN == 0 && m % kVecN == 0)
<< "gelu_tanh_cat: both widths must be positive multiples of " << kVecN;
CHECK_HOST(rows.unwrap() > 0) << "gelu_tanh_cat: rows must be positive";
CHECK_HOST(
reinterpret_cast<uintptr_t>(attn.data_ptr()) % 16 == 0 && reinterpret_cast<uintptr_t>(mlp.data_ptr()) % 16 == 0 &&
reinterpret_cast<uintptr_t>(output.data_ptr()) % 16 == 0)
<< "gelu_tanh_cat: tensors must be 16-byte aligned";
const int64_t n = rows.unwrap() * ((a + m) / kVecN);
CHECK_HOST(n <= std::numeric_limits<int32_t>::max()) << "gelu_tanh_cat: tensor is too large";
constexpr int kBlockSize = 256;
const auto kernel = gelu_tanh_cat_kernel<T, kVecN>;
const int64_t occupancy = runtime::get_blocks_per_sm(kernel, kBlockSize);
const int64_t sms = runtime::get_sm_count(device_.unwrap().device_id);
const int64_t grid = std::min(sms * occupancy, div_ceil(n, int64_t{kBlockSize}));
LaunchKernel(grid, kBlockSize, device_.unwrap())(
kernel,
static_cast<const T*>(attn.data_ptr()),
static_cast<const T*>(mlp.data_ptr()),
static_cast<T*>(output.data_ptr()),
static_cast<uint32_t>(n),
static_cast<uint32_t>(a / kVecN),
static_cast<uint32_t>(m / kVecN));
}
} // namespace sglang
@@ -49,6 +49,13 @@ _HIP = frozenset({CapabilityRequirement.HIP})
# inventory: callers name the one they want via ``select_kernel``.
# ---------------------------------------------------------------------------
_SPECS: tuple[tuple[str, KernelBackend, str, frozenset, str], ...] = (
(
"diffusion.gelu_tanh_cat",
KernelBackend.JIT,
"activation.gelu_tanh_cat_jit:fused_gelu_tanh_cat",
_CUDA,
"BF16 tanh GELU plus channel concatenation.",
),
(
"diffusion.apply_group_norm_silu",
KernelBackend.TRITON,
@@ -527,6 +534,8 @@ for _op, _backend, _target, _caps, _description in _SPECS:
# then symbol; a new public kernel belongs here and nowhere else.
# ---------------------------------------------------------------------------
_EXPORTS: dict[str, str] = {
"can_use_fused_gelu_tanh_cat": "activation.gelu_tanh_cat_jit",
"fused_gelu_tanh_cat": "activation.gelu_tanh_cat_jit",
"load_extension_with_recovery": "sglang.srt.utils.cpp_extension_loader",
# Normalization: RMSNorm / LayerNorm / GroupNorm and their fused epilogues
"can_defer_flux2_gated_residual": "norm.flux2_gated_resnorm_jit",
@@ -0,0 +1,64 @@
"""Concatenate attention with tanh GELU while retaining the BF16 rounding boundary.
The inputs are contiguous BF16 tensors with equal leading dimensions and
16-byte aligned rows. The result matches ``cat((attn, gelu(mlp)), dim=-1)``.
"""
from __future__ import annotations
import torch
from sglang.kernels.jit.utils import cache_once, load_jit, make_cpp_args
from sglang.srt.utils.custom_op import register_custom_op
@cache_once
def _module():
args = make_cpp_args(torch.bfloat16)
return load_jit(
"gelu_tanh_cat",
*args,
cuda_files=["diffusion/gelu_tanh_cat.cuh"],
cuda_wrappers=[("gelu_tanh_cat", f"gelu_tanh_cat<{args}>")],
extra_cuda_cflags=["-lineinfo"],
)
def can_use_fused_gelu_tanh_cat(attn: torch.Tensor, mlp: torch.Tensor) -> bool:
return (
not torch.is_grad_enabled()
and not torch.compiler.is_compiling()
and torch.version.hip is None
and attn.is_cuda
and attn.dtype == mlp.dtype == torch.bfloat16
and attn.device == mlp.device
and attn.ndim >= 2
and attn.shape[:-1] == mlp.shape[:-1]
and attn.shape[-1] > 0
and mlp.shape[-1] > 0
and attn.shape[-1] % 8 == mlp.shape[-1] % 8 == 0
and attn.is_contiguous()
and mlp.is_contiguous()
and attn.numel() > 0
and (attn.numel() + mlp.numel()) // 8 <= 2**31 - 1
and attn.data_ptr() % 16 == mlp.data_ptr() % 16 == 0
)
@register_custom_op(mutates_args=["output"])
def _gelu_tanh_cat(attn: torch.Tensor, mlp: torch.Tensor, output: torch.Tensor) -> None:
_module().gelu_tanh_cat(
attn.view(-1, attn.shape[-1]),
mlp.view(-1, mlp.shape[-1]),
output.view(-1, output.shape[-1]),
)
def fused_gelu_tanh_cat(attn: torch.Tensor, mlp: torch.Tensor) -> torch.Tensor:
output = torch.empty(
(*attn.shape[:-1], attn.shape[-1] + mlp.shape[-1]),
device=attn.device,
dtype=attn.dtype,
)
_gelu_tanh_cat(attn, mlp, output)
return output
@@ -40,6 +40,23 @@ class LongCatImageEditSamplingParams(SamplingParams):
enable_cfg_renorm: bool = False
enable_prompt_rewrite: bool = False
def prepare_synthetic_warmup_request_for_queue(self, req, server_args) -> None:
if server_args.warmup_resolutions is None:
return
# Editing derives both the output grid and VL prefix from the input
# image. A square placeholder discards an explicit non-square warmup
# resolution and captures a graph that the real request cannot replay.
import base64
import io
from PIL import Image
buffer = io.BytesIO()
Image.new("RGB", (req.width, req.height)).save(buffer, format="PNG")
req.image_path = [
"data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode()
]
@dataclass
class LongCatImageEditTurboSamplingParams(LongCatImageEditSamplingParams):
@@ -23,8 +23,10 @@ def is_longcat_image_transformer(current_model: Any, call_kwargs: dict) -> bool:
def keep_longcat_prompt_shape(
call_kwargs: dict, _current_model: Any, _buckets: tuple[int, ...]
) -> dict:
# LongCat always supplies the complete 512-token prompt body to its DiT.
# Generic bucket padding would only create larger, unused graph signatures.
# LongCat supplies the complete 512-token prompt body to its DiT. Editing
# also prepends VL image tokens, whose count depends on the input shape.
# Preserve this exact sequence and the corresponding txt_ids; padding
# would change the attention/RoPE semantics for the unmasked joint sequence.
return call_kwargs
@@ -64,6 +64,30 @@ logger = init_logger(__name__)
_LONGCAT_QKNORM_ROPE = BitExactFusionGate("LongCat fused QKNorm+RoPE")
_LONGCAT_LN_MOD = BitExactFusionGate("LongCat fused LN+modulate", per_signature=True)
_LONGCAT_GELU_CAT = BitExactFusionGate("LongCat fused GELU+cat")
def _longcat_gelu_cat(
attn: torch.Tensor, mlp: torch.Tensor, activation: nn.Module
) -> torch.Tensor:
if (
type(activation) is nn.GELU
and activation.approximate == "tanh"
and _LONGCAT_GELU_CAT.can_attempt_once()
and diffusion_ops.can_use_fused_gelu_tanh_cat(attn, mlp)
):
try:
output = diffusion_ops.fused_gelu_tanh_cat(attn, mlp)
except Exception as exc:
_LONGCAT_GELU_CAT.on_exception(exc, logger=logger)
else:
if _LONGCAT_GELU_CAT.verified:
return output
reference = torch.cat((attn, activation(mlp)), dim=-1)
return _LONGCAT_GELU_CAT.accept_or_fallback(
output, reference, logger=logger
)
return torch.cat((attn, activation(mlp)), dim=-1)
def _longcat_norm_modulate(
@@ -649,15 +673,15 @@ class _SingleTransformerBlock(nn.Module):
residual = hidden_states
norm_hidden_states, gate = self.norm(hidden_states, emb=temb)
if fused_gelu_active(self) and can_use_linear_gelu(
gelu_in_projection = fused_gelu_active(self) and can_use_linear_gelu(
self.proj_mlp, norm_hidden_states
):
)
if gelu_in_projection:
mlp_hidden_states = fused_linear_gelu_tanh(
norm_hidden_states, self.proj_mlp.weight, self.proj_mlp.bias
)
else:
mlp_hidden_states, _ = self.proj_mlp(norm_hidden_states)
mlp_hidden_states = self.act_mlp(mlp_hidden_states)
attn_output = self.attn(
hidden_states=norm_hidden_states,
image_rotary_emb=image_rotary_emb,
@@ -666,7 +690,12 @@ class _SingleTransformerBlock(nn.Module):
cos_sin_cache=cos_sin_cache,
positions=positions,
)
hidden_states = torch.cat([attn_output, mlp_hidden_states], dim=2)
if gelu_in_projection:
hidden_states = torch.cat([attn_output, mlp_hidden_states], dim=2)
else:
hidden_states = _longcat_gelu_cat(
attn_output, mlp_hidden_states, self.act_mlp
)
gate = gate.unsqueeze(1)
hidden_states, _ = self.proj_out(hidden_states)
hidden_states = residual_gate_add(residual, hidden_states, gate)
@@ -212,6 +212,7 @@ BREAKABLE_CUDA_GRAPH_SUPPORTED_MODEL_IDS = frozenset(
"lightricks/ltx-2",
"lightricks/ltx-2.3",
"meituan-longcat/longcat-image",
"meituan-longcat/longcat-image-edit-turbo",
"ltx-2",
"ltx-2.3",
"minimax-h3",
@@ -239,6 +240,7 @@ BREAKABLE_CUDA_GRAPH_SUPPORTED_PIPELINE_CONFIGS = frozenset(
"LTX2PipelineConfig",
"LTX23PipelineConfig",
"LongCatImagePipelineConfig",
"LongCatImageEditPipelineConfig",
"MiniMaxH3PipelineConfig",
"QwenImagePipelineConfig",
"QwenImage21PipelineConfig",
@@ -225,6 +225,23 @@ class TestDiffusionBCGPadding(unittest.TestCase):
self.assertEqual(out["encoder_hidden_states"].shape, (1, 512, 3584))
self.assertEqual(out["txt_ids"].shape, (512, 3))
def test_longcat_edit_keeps_vl_prefix_and_image_grid(self):
kwargs = {
"hidden_states": torch.zeros(1, 8374, 64),
"encoder_hidden_states": torch.zeros(1, 859, 64),
"encoder_hidden_states_mask": [torch.ones(1, 859, dtype=torch.bool)],
"txt_ids": torch.zeros(859, 3),
"img_ids": torch.zeros(8374, 3),
}
with self._patch_buckets(64, 128, 256, 512, 1024):
out = self.stage._bcg_pad_prompt_kwargs(
kwargs, current_model=self.longcat_model
)
self.assertIs(out, kwargs)
self.assertEqual(out["encoder_hidden_states"].shape[1], 859)
self.assertEqual(out["txt_ids"].shape, (859, 3))
self.assertEqual(out["img_ids"].shape, (8374, 3))
def test_qwen_default_bucket_preserves_mask(self):
def kwargs(valid_len: int):
mask = torch.zeros(1, 64, dtype=torch.bool)
@@ -473,6 +490,7 @@ class TestDiffusionBCGPadding(unittest.TestCase):
def test_image_generation_models_are_registered_as_bcg_supported(self):
for model_id in (
"meituan-longcat/longcat-image",
"meituan-longcat/longcat-image-edit-turbo",
"qwen/qwen-image",
"qwen/qwen-image-2512",
"tongyi-mai/z-image",
@@ -484,6 +502,7 @@ class TestDiffusionBCGPadding(unittest.TestCase):
for config_name in (
"GlmImagePipelineConfig",
"LongCatImagePipelineConfig",
"LongCatImageEditPipelineConfig",
"QwenImagePipelineConfig",
"ZImagePipelineConfig",
):
@@ -11,6 +11,10 @@ from sglang.multimodal_gen.configs.pipeline_configs.longcat_image import (
LongCatImagePipelineConfig,
_calculate_edit_dimensions,
)
from sglang.multimodal_gen.configs.sample.longcat_image import (
LongCatImageEditSamplingParams,
LongCatImageEditTurboSamplingParams,
)
@pytest.fixture
@@ -37,6 +41,38 @@ def test_edit_dimensions_match_diffusers_formula():
assert got_w % 16 == 0 and got_h % 16 == 0
@pytest.mark.parametrize(
"params_cls", [LongCatImageEditSamplingParams, LongCatImageEditTurboSamplingParams]
)
@pytest.mark.parametrize("size", [(1264, 848), (848, 1264), (1024, 1024)])
def test_explicit_warmup_image_preserves_edit_grids(params_cls, size):
import base64
import io
from PIL import Image
params = params_cls()
req = _make_batch(width=size[0], height=size[1], image_path=["placeholder.png"])
server_args = _make_batch(warmup_resolutions=[f"{size[0]}x{size[1]}"])
params.prepare_synthetic_warmup_request_for_queue(req, server_args)
image = Image.open(io.BytesIO(base64.b64decode(req.image_path[0].split(",", 1)[1])))
assert image.size == size
config = LongCatImageEditPipelineConfig()
assert config.calculate_condition_image_size(image, *image.size) == size
ids = config._edit_img_ids(
_make_batch(width=size[0], height=size[1]), 859, torch.device("cpu")
)
assert ids.shape == (2 * (size[0] // 16) * (size[1] // 16), 3)
def test_default_warmup_keeps_existing_placeholder():
req = _make_batch(width=512, height=512, image_path=["placeholder.png"])
LongCatImageEditSamplingParams().prepare_synthetic_warmup_request_for_queue(
req, _make_batch(warmup_resolutions=None)
)
assert req.image_path == ["placeholder.png"]
def test_edit_config_task_type_and_generator(edit_config):
assert edit_config.task_type == ModelTaskType.I2I
assert edit_config.generator_device == "cpu"